From b355ccec9eef7ce0afce765955b8a74f9a8e5c08 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:59:20 +0100 Subject: [PATCH] Add Valkey cluster backplane and sticky-410 ownership (clusters) (#6472) --- .../common/cluster/RateLimitStore.java | 8 +- .../common/service/JobExecutorService.java | 135 +---- .../InProcessDistributedLockTest.java | 4 +- .../TaskManagerJobStoreDelegationTest.java | 2 + .../common/controller/JobController.java | 219 +++++--- .../common/controller/JobOwnershipCache.java | 47 ++ .../JobControllerOwnershipTest.java | 479 ++++++++++++++++ .../common/controller/JobControllerTest.java | 6 + app/proprietary/build.gradle | 12 +- .../cluster/ClusterLicenseGate.java | 40 ++ .../proprietary/cluster/ClusterMetrics.java | 113 ++++ .../cluster/ClusterNodeBootstrap.java | 201 +++++++ .../valkey/ConditionalOnValkeyBackplane.java | 19 + .../valkey/ValkeyClusterBackplane.java | 55 ++ .../valkey/ValkeyConnectionConfiguration.java | 265 +++++++++ .../cluster/valkey/ValkeyDistributedLock.java | 102 ++++ .../valkey/ValkeyInstanceRegistry.java | 115 ++++ .../cluster/valkey/ValkeyJobStore.java | 297 ++++++++++ .../cluster/valkey/ValkeyKeyValueCache.java | 61 ++ .../cluster/valkey/ValkeyRateLimitStore.java | 83 +++ .../cluster/ClusterLicenseGateTest.java | 62 +++ .../cluster/ClusterMetricsTest.java | 122 ++++ .../cluster/ClusterNodeBootstrapTest.java | 115 ++++ .../cluster/MultiNodeClusterScenarioTest.java | 133 +++++ .../valkey/LiveExternalClusterTest.java | 237 ++++++++ .../valkey/LiveValkeyAuthIntegrationTest.java | 143 +++++ .../cluster/valkey/LiveValkeyChaosTest.java | 118 ++++ .../valkey/LiveValkeyIntegrationTest.java | 519 ++++++++++++++++++ .../valkey/ValkeyClusterBackplaneTest.java | 61 ++ .../ValkeyConnectionConfigurationTest.java | 298 ++++++++++ 30 files changed, 3858 insertions(+), 213 deletions(-) create mode 100644 app/core/src/main/java/stirling/software/common/controller/JobOwnershipCache.java create mode 100644 app/core/src/test/java/stirling/software/common/controller/JobControllerOwnershipTest.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterLicenseGate.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterMetrics.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterNodeBootstrap.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ConditionalOnValkeyBackplane.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplane.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfiguration.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyDistributedLock.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyInstanceRegistry.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyJobStore.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyKeyValueCache.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyRateLimitStore.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterLicenseGateTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterMetricsTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterNodeBootstrapTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/MultiNodeClusterScenarioTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveExternalClusterTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyAuthIntegrationTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyChaosTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyIntegrationTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplaneTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfigurationTest.java diff --git a/app/common/src/main/java/stirling/software/common/cluster/RateLimitStore.java b/app/common/src/main/java/stirling/software/common/cluster/RateLimitStore.java index 82351961a..21cb31e12 100644 --- a/app/common/src/main/java/stirling/software/common/cluster/RateLimitStore.java +++ b/app/common/src/main/java/stirling/software/common/cluster/RateLimitStore.java @@ -2,7 +2,13 @@ package stirling.software.common.cluster; import java.time.Duration; -/** Token-bucket rate limiting backed by the cluster backplane. */ +/** + * Token-bucket rate limiting backed by the cluster backplane. + * + *

In-process implementations enforce a per-JVM limit; distributed implementations enforce a + * single global limit across every node. Both use a Bucket4j greedy-refill token bucket so the + * semantics match across single-node and cluster deployments. + */ public interface RateLimitStore { /** diff --git a/app/common/src/main/java/stirling/software/common/service/JobExecutorService.java b/app/common/src/main/java/stirling/software/common/service/JobExecutorService.java index 4c5112932..629d28ba6 100644 --- a/app/common/src/main/java/stirling/software/common/service/JobExecutorService.java +++ b/app/common/src/main/java/stirling/software/common/service/JobExecutorService.java @@ -57,85 +57,38 @@ public class JobExecutorService { this.resourceMonitor = resourceMonitor; this.jobQueue = jobQueue; - // Parse session timeout and calculate effective timeout once during initialization long sessionTimeoutMs = parseSessionTimeout(sessionTimeout); this.effectiveTimeoutMs = Math.min(asyncRequestTimeoutMs, sessionTimeoutMs); log.debug( "Job executor configured with effective timeout of {} ms", this.effectiveTimeoutMs); } - /** - * Run a job either asynchronously or synchronously - * - * @param async Whether to run the job asynchronously - * @param work The work to be done - * @return The response - */ public ResponseEntity runJobGeneric(boolean async, Supplier work) { return runJobGeneric(async, work, -1); } - /** - * Run a job either asynchronously or synchronously with a custom timeout - * - * @param async Whether to run the job asynchronously - * @param work The work to be done - * @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default - * @return The response - */ public ResponseEntity runJobGeneric( boolean async, Supplier work, long customTimeoutMs) { return runJobGeneric(async, work, customTimeoutMs, false, 50); } - /** - * Run a job either asynchronously or synchronously with custom parameters - * - * @param async Whether to run the job asynchronously - * @param work The work to be done - * @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default - * @param queueable Whether this job can be queued when system resources are limited - * @param resourceWeight The resource weight of this job (1-100) - * @return The response - */ public ResponseEntity runJobGeneric( boolean async, Supplier work, long customTimeoutMs, boolean queueable, int resourceWeight) { - // Generate base UUID String baseJobId = UUID.randomUUID().toString(); - - // Scope job to authenticated user if security is enabled String scopedJobKey = getScopedJobKey(baseJobId); log.debug("Generated jobId: {} (base: {})", scopedJobKey, baseJobId); - // Store the scoped job ID in the request for potential use by other components if (request != null) { request.setAttribute("jobId", scopedJobKey); - - // Also track this job ID in the user's session for authorization purposes - // This ensures users can only cancel their own jobs - if (request.getSession() != null) { - @SuppressWarnings("unchecked") - java.util.Set userJobIds = - (java.util.Set) request.getSession().getAttribute("userJobIds"); - - if (userJobIds == null) { - userJobIds = new java.util.concurrent.ConcurrentSkipListSet<>(); - request.getSession().setAttribute("userJobIds", userJobIds); - } - - userJobIds.add(scopedJobKey); - log.debug("Added scoped job ID {} to user session", scopedJobKey); - } } String jobId = scopedJobKey; - // Determine which timeout to use long timeoutToUse = customTimeoutMs > 0 ? customTimeoutMs : effectiveTimeoutMs; log.debug( @@ -146,7 +99,6 @@ public class JobExecutorService { queueable, resourceWeight); - // Check if we need to queue this job based on resource availability boolean shouldQueue = queueable && async @@ -154,7 +106,6 @@ public class JobExecutorService { resourceMonitor.shouldQueueJob(resourceWeight); if (shouldQueue) { - // Queue the job instead of executing immediately log.debug( "Queueing job {} due to resource constraints (weight: {})", jobId, @@ -162,18 +113,12 @@ public class JobExecutorService { taskManager.createTask(jobId); - // Create a specialized wrapper that updates the TaskManager final String capturedJobIdForQueue = jobId; Supplier wrappedWork = () -> { try { - // Set jobId in ThreadLocal context for the queued job stirling.software.common.util.JobContext.setJobId( capturedJobIdForQueue); - log.debug( - "Set jobId {} in JobContext for queued job execution", - capturedJobIdForQueue); - Object result = work.get(); processJobResult(capturedJobIdForQueue, result); return result; @@ -186,21 +131,17 @@ public class JobExecutorService { taskManager.setError(capturedJobIdForQueue, e.getMessage()); throw e; } finally { - // Clean up ThreadLocal to avoid memory leaks stirling.software.common.util.JobContext.clear(); } }; - // Queue the job and get the future CompletableFuture> future = jobQueue.queueJob(jobId, resourceWeight, wrappedWork, timeoutToUse); - // Return immediately with job ID return ResponseEntity.ok().body(new JobResponse<>(true, jobId, null)); } else if (async) { taskManager.createTask(jobId); - // Capture the jobId for the async thread final String capturedJobId = jobId; executor.execute( @@ -211,13 +152,7 @@ public class JobExecutorService { capturedJobId, timeoutToUse); - // Set jobId in ThreadLocal context for the async thread stirling.software.common.util.JobContext.setJobId(capturedJobId); - log.debug( - "Set jobId {} in JobContext for async execution", - capturedJobId); - - // Execute with timeout Object result = executeWithTimeout(() -> work.get(), timeoutToUse); processJobResult(capturedJobId, result); } catch (TimeoutException te) { @@ -227,7 +162,6 @@ public class JobExecutorService { log.error("Error executing job {}: {}", jobId, e.getMessage(), e); taskManager.setError(jobId, e.getMessage()); } finally { - // Clean up ThreadLocal to avoid memory leaks stirling.software.common.util.JobContext.clear(); } }); @@ -237,27 +171,19 @@ public class JobExecutorService { try { log.debug("Running sync job with timeout {} ms", timeoutToUse); - // Make jobId available to downstream components on the worker thread stirling.software.common.util.JobContext.setJobId(jobId); - log.debug("Set jobId {} in JobContext for sync execution", jobId); - - // Execute with timeout Object result = executeWithTimeout(() -> work.get(), timeoutToUse); - // If the result is already a ResponseEntity, return it directly if (result instanceof ResponseEntity) { return (ResponseEntity) result; } - // Process different result types return handleResultForSyncJob(result); } catch (TimeoutException te) { log.error("Synchronous job timed out after {} ms", timeoutToUse); return ResponseEntity.internalServerError() .body(Map.of("error", "Job timed out after " + timeoutToUse + " ms")); } catch (RuntimeException e) { - // Check if this is a typed exception that should be handled by - // GlobalExceptionHandler (either directly or wrapped) Throwable cause = e.getCause(); if (e instanceof IllegalArgumentException || cause @@ -267,16 +193,13 @@ public class JobExecutorService { instanceof stirling.software.common.util.ExceptionUtils .BaseValidationException) { - // Rethrow so GlobalExceptionHandler can handle with proper HTTP status codes throw e; } - // Handle other RuntimeExceptions as generic errors log.error("Error executing synchronous job: {}", e.getMessage(), e); return ResponseEntity.internalServerError() .body(Map.of("error", "Job failed: " + e.getMessage())); } catch (Exception e) { log.error("Error executing synchronous job: {}", e.getMessage(), e); - // Construct a JSON error response return ResponseEntity.internalServerError() .body(Map.of("error", "Job failed: " + e.getMessage())); } finally { @@ -285,23 +208,13 @@ public class JobExecutorService { } } - /** - * Process the result of an asynchronous job - * - * @param jobId The job ID - * @param result The result - */ private void processJobResult(String jobId, Object result) { try { if (result instanceof byte[]) { - // Store byte array directly to disk to avoid double memory consumption String fileId = fileStorage.storeBytes((byte[]) result, "result.pdf"); taskManager.setFileResult( jobId, fileId, "result.pdf", MediaType.APPLICATION_PDF_VALUE); log.debug("Stored byte[] result with fileId: {}", fileId); - - // Let the byte array get collected naturally in the next GC cycle - // We don't need to force System.gc() which can be harmful } else if (result instanceof ResponseEntity) { ResponseEntity response = (ResponseEntity) result; Object body = response.getBody(); @@ -330,16 +243,13 @@ public class JobExecutorService { taskManager.setFileResult(jobId, fileId, filename, contentType); log.debug("Stored ResponseEntity result with fileId: {}", fileId); } else { - // Check if the response body contains a fileId if (body != null && body.toString().contains("fileId")) { try { - // Try to extract fileId using reflection java.lang.reflect.Method getFileId = body.getClass().getMethod("getFileId"); String fileId = (String) getFileId.invoke(body); if (fileId != null && !fileId.isEmpty()) { - // Try to get filename and content type String filename = "result.pdf"; String contentType = MediaType.APPLICATION_PDF_VALUE; @@ -379,7 +289,6 @@ public class JobExecutorService { } } - // Store generic result taskManager.setResult(jobId, body); } } else if (result instanceof MultipartFile file) { @@ -388,16 +297,13 @@ public class JobExecutorService { jobId, fileId, file.getOriginalFilename(), file.getContentType()); log.debug("Stored MultipartFile result with fileId: {}", fileId); } else { - // Check if result has a fileId field if (result != null) { try { - // Try to extract fileId using reflection java.lang.reflect.Method getFileId = result.getClass().getMethod("getFileId"); String fileId = (String) getFileId.invoke(result); if (fileId != null && !fileId.isEmpty()) { - // Try to get filename and content type String filename = "result.pdf"; String contentType = MediaType.APPLICATION_PDF_VALUE; @@ -435,7 +341,6 @@ public class JobExecutorService { } } - // Default case: store the result as is taskManager.setResult(jobId, result); } @@ -446,16 +351,8 @@ public class JobExecutorService { } } - /** - * Handle different result types for synchronous jobs - * - * @param result The result object - * @return The appropriate ResponseEntity - * @throws IOException If there is an error processing the result - */ private ResponseEntity handleResultForSyncJob(Object result) throws IOException { if (result instanceof byte[]) { - // Return byte array as PDF return ResponseEntity.ok() .contentType(MediaType.APPLICATION_PDF) .header( @@ -463,7 +360,6 @@ public class JobExecutorService { "form-data; name=\"attachment\"; filename=\"result.pdf\"") .body(result); } else if (result instanceof MultipartFile file) { - // Return MultipartFile content return ResponseEntity.ok() .contentType(MediaType.parseMediaType(file.getContentType())) .header( @@ -473,7 +369,6 @@ public class JobExecutorService { + "\"") .body(file.getBytes()); } else { - // Default case: return as JSON return ResponseEntity.ok(result); } } @@ -493,15 +388,9 @@ public class JobExecutorService { return mediaType != null ? mediaType.toString() : MediaType.APPLICATION_PDF_VALUE; } - /** - * Parse session timeout string (e.g., "30m", "1h") to milliseconds - * - * @param timeout The timeout string - * @return The timeout in milliseconds - */ private long parseSessionTimeout(String timeout) { if (timeout == null || timeout.isEmpty()) { - return 30 * 60 * 1000; // Default: 30 minutes + return 30 * 60 * 1000; } try { @@ -523,27 +412,16 @@ public class JobExecutorService { case "m" -> (long) (numericValue * 60 * 1000); case "h" -> (long) (numericValue * 60 * 60 * 1000); case "d" -> (long) (numericValue * 24 * 60 * 60 * 1000); - default -> (long) (numericValue * 60 * 1000); // Default to minutes + default -> (long) (numericValue * 60 * 1000); }; } catch (Exception e) { log.warn("Could not parse session timeout '{}', using default", timeout); - return 30 * 60 * 1000; // Default: 30 minutes + return 30 * 60 * 1000; } } - /** - * Execute a supplier with a timeout - * - * @param supplier The supplier to execute - * @param timeoutMs The timeout in milliseconds - * @return The result from the supplier - * @throws TimeoutException If the execution times out - * @throws Exception If the supplier throws an exception - */ private T executeWithTimeout(Supplier supplier, long timeoutMs) throws TimeoutException, Exception { - // Use the same executor as other async jobs for consistency - // This ensures all operations run on the same thread pool String currentJobId = stirling.software.common.util.JobContext.getJobId(); java.util.concurrent.CompletableFuture future = @@ -577,17 +455,10 @@ public class JobExecutorService { } } - /** - * Get a scoped job key that includes user ownership when security is enabled. - * - * @param baseJobId the base job identifier - * @return scoped job key, or just baseJobId if no ownership service available - */ private String getScopedJobKey(String baseJobId) { if (jobOwnershipService != null) { return jobOwnershipService.createScopedJobKey(baseJobId); } - // Security disabled, return unsecured job key return baseJobId; } } diff --git a/app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessDistributedLockTest.java b/app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessDistributedLockTest.java index 30b738a3b..2656c0d09 100644 --- a/app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessDistributedLockTest.java +++ b/app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessDistributedLockTest.java @@ -24,7 +24,9 @@ class InProcessDistributedLockTest { } @Test - void reentryFromSameThreadFails() { + void reentryFromSameThreadFails_parityWithValkey() { + // The Valkey impl refuses reentry (SET NX semantics); the in-process impl must match, + // otherwise code working in single-instance silently breaks in cluster mode. DistributedLock lock = new InProcessDistributedLock(); DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow(); Optional reentry = lock.tryAcquire("k", Duration.ofSeconds(30)); diff --git a/app/common/src/test/java/stirling/software/common/service/TaskManagerJobStoreDelegationTest.java b/app/common/src/test/java/stirling/software/common/service/TaskManagerJobStoreDelegationTest.java index 1316c3c13..23165a14a 100644 --- a/app/common/src/test/java/stirling/software/common/service/TaskManagerJobStoreDelegationTest.java +++ b/app/common/src/test/java/stirling/software/common/service/TaskManagerJobStoreDelegationTest.java @@ -86,6 +86,8 @@ class TaskManagerJobStoreDelegationTest { @Override public boolean shouldRunLocalCleanup() { + // Distributed backplanes own job TTL eviction themselves; this mock + // mirrors the real ValkeyClusterBackplane override of the default true. return false; } }; diff --git a/app/core/src/main/java/stirling/software/common/controller/JobController.java b/app/core/src/main/java/stirling/software/common/controller/JobController.java index 7bd81741e..bab5dce53 100644 --- a/app/core/src/main/java/stirling/software/common/controller/JobController.java +++ b/app/core/src/main/java/stirling/software/common/controller/JobController.java @@ -4,6 +4,7 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; +import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; @@ -22,6 +23,10 @@ import jakarta.servlet.http.HttpServletRequest; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import stirling.software.common.cluster.ClusterBackplane; +import stirling.software.common.cluster.JobStore; +import stirling.software.common.cluster.JobStoreEntry; +import stirling.software.common.cluster.StickyMissRecorder; import stirling.software.common.model.job.JobResult; import stirling.software.common.model.job.ResultFile; import stirling.software.common.service.FileStorage; @@ -30,7 +35,6 @@ import stirling.software.common.service.JobQueue; import stirling.software.common.service.TaskManager; import stirling.software.common.util.RegexPatternUtils; -/** REST controller for job-related endpoints */ @RestController @RequiredArgsConstructor @Slf4j @@ -42,20 +46,29 @@ public class JobController { private final FileStorage fileStorage; private final JobQueue jobQueue; private final HttpServletRequest request; + private final ClusterBackplane clusterBackplane; + private final JobStore jobStore; + + // Short-TTL local cache fronting JobStore.get() on the sticky-410 path to avoid a Valkey + // HGETALL round-trip on every download retry for the same job. + private final JobOwnershipCache ownershipCache = new JobOwnershipCache(); @Autowired(required = false) private JobOwnershipService jobOwnershipService; - /** - * Get the status of a job - * - * @param jobId The job ID - * @return The job result - */ + @Autowired(required = false) + private StickyMissRecorder stickyMissRecorder; + @GetMapping("/job/{jobId}") @Operation(summary = "Get job status") public ResponseEntity getJobStatus(@PathVariable("jobId") String jobId) { - // Validate job ownership + // Sticky-410 must run before user-auth: a 403 here would leak job existence and defeat + // LB re-routing. The owner node is where the real auth check should happen. + Optional> peerOwned = guardNonOwner(jobId); + if (peerOwned.isPresent()) { + return peerOwned.get(); + } + if (!validateJobAccess(jobId)) { log.warn("Unauthorized attempt to access job status: {}", jobId); return ResponseEntity.status(403) @@ -67,7 +80,6 @@ public class JobController { return ResponseEntity.notFound().build(); } - // Check if the job is in the queue and add queue information if (!result.isComplete() && jobQueue.isJobQueued(jobId)) { int position = jobQueue.getJobPosition(jobId); Map resultWithQueueInfo = @@ -82,16 +94,14 @@ public class JobController { return ResponseEntity.ok(result); } - /** - * Get the result of a job - * - * @param jobId The job ID - * @return The job result - */ @GetMapping("/job/{jobId}/result") @Operation(summary = "Get job result") public ResponseEntity getJobResult(@PathVariable("jobId") String jobId) { - // Validate job ownership + Optional> peerOwned = guardNonOwner(jobId); + if (peerOwned.isPresent()) { + return peerOwned.get(); + } + if (!validateJobAccess(jobId)) { log.warn("Unauthorized attempt to access job result: {}", jobId); return ResponseEntity.status(403) @@ -111,7 +121,6 @@ public class JobController { return ResponseEntity.badRequest().body("Job failed: " + result.getError()); } - // Handle multiple files - return metadata for client to download individually if (result.hasMultipleFiles()) { return ResponseEntity.ok() .contentType(MediaType.APPLICATION_JSON) @@ -125,11 +134,11 @@ public class JobController { result.getAllResultFiles())); } - // Handle single file (download directly) if (result.hasFiles() && !result.hasMultipleFiles()) { try { List files = result.getAllResultFiles(); ResultFile singleFile = files.get(0); + byte[] fileContent = fileStorage.retrieveBytes(singleFile.getFileId()); return ResponseEntity.ok() .header("Content-Type", singleFile.getContentType()) @@ -147,30 +156,22 @@ public class JobController { return ResponseEntity.ok(result.getResult()); } - // Admin-only endpoints have been moved to AdminJobController in the proprietary package - - /** - * Cancel a job by its ID - * - *

This method should only allow cancellation of jobs that were created by the current user. - * The jobId should be part of the user's session or otherwise linked to their identity. - * - * @param jobId The job ID - * @return Response indicating whether the job was cancelled - */ @DeleteMapping("/job/{jobId}") @Operation(summary = "Cancel a job") public ResponseEntity cancelJob(@PathVariable("jobId") String jobId) { log.debug("Request to cancel job: {}", jobId); - // Validate job ownership + Optional> peerOwned = guardNonOwner(jobId); + if (peerOwned.isPresent()) { + return peerOwned.get(); + } + if (!validateJobAccess(jobId)) { log.warn("Unauthorized attempt to cancel job: {}", jobId); return ResponseEntity.status(403) .body(Map.of("message", "You are not authorized to cancel this job")); } - // First check if the job is in the queue boolean cancelled = false; int queuePosition = -1; @@ -180,11 +181,9 @@ public class JobController { log.info("Cancelled queued job: {} (was at position {})", jobId, queuePosition); } - // If not in queue or couldn't cancel, try to cancel in TaskManager if (!cancelled) { JobResult result = taskManager.getJobResult(jobId); if (result != null && !result.isComplete()) { - // Mark as error with cancellation message taskManager.setError(jobId, "Job was cancelled by user"); cancelled = true; log.info("Marked job as cancelled in TaskManager: {}", jobId); @@ -201,7 +200,6 @@ public class JobController { "queuePosition", queuePosition >= 0 ? queuePosition : "n/a")); } else { - // Job not found or already complete JobResult result = taskManager.getJobResult(jobId); if (result == null) { return ResponseEntity.notFound().build(); @@ -215,16 +213,14 @@ public class JobController { } } - /** - * Get the list of files for a job - * - * @param jobId The job ID - * @return List of files for the job - */ @GetMapping("/job/{jobId}/result/files") @Operation(summary = "Get job result files") public ResponseEntity getJobFiles(@PathVariable("jobId") String jobId) { - // Validate job ownership + Optional> peerOwned = guardNonOwner(jobId); + if (peerOwned.isPresent()) { + return peerOwned.get(); + } + if (!validateJobAccess(jobId)) { log.warn("Unauthorized attempt to access job files: {}", jobId); return ResponseEntity.status(403) @@ -252,28 +248,31 @@ public class JobController { "files", files)); } - /** - * Get metadata for an individual file by its file ID - * - * @param fileId The file ID - * @return The file metadata - */ @GetMapping("/files/{fileId}/metadata") @Operation(summary = "Get file metadata") public ResponseEntity getFileMetadata(@PathVariable("fileId") String fileId) { try { - String jobKey = taskManager.findJobKeyByFileId(fileId); + String jobKey; + try { + jobKey = taskManager.findJobKeyByFileId(fileId); + } catch (RuntimeException backplaneEx) { + return backplaneUnavailable(fileId, backplaneEx); + } if (jobKey == null) { return ResponseEntity.notFound().build(); } + Optional> notOwner = guardNonOwner(jobKey); + if (notOwner.isPresent()) { + return notOwner.get(); + } + if (!validateJobAccess(jobKey)) { log.warn("Unauthorized attempt to access file metadata: {}", fileId); return ResponseEntity.status(403) .body(Map.of("message", "You are not authorized to access this file")); } - // Find the file metadata from any job that contains this file ResultFile resultFile = taskManager.findResultFileByFileId(fileId); if (resultFile != null) { @@ -281,12 +280,10 @@ public class JobController { } if (!isSecurityEnabled()) { - // Backwards compatibility when ownership service is unavailable if (!fileStorage.fileExists(fileId)) { return ResponseEntity.notFound().build(); } - // File exists but no metadata found, get basic info efficiently long fileSize = fileStorage.getFileSize(fileId); return ResponseEntity.ok( Map.of( @@ -308,32 +305,31 @@ public class JobController { } } - /** - * Download an individual file by its file ID - * - * @param fileId The file ID - * @return The file content - */ @GetMapping("/files/{fileId}") @Operation(summary = "Download a file") public ResponseEntity downloadFile(@PathVariable("fileId") String fileId) { try { - String jobKey = taskManager.findJobKeyByFileId(fileId); + String jobKey; + try { + jobKey = taskManager.findJobKeyByFileId(fileId); + } catch (RuntimeException backplaneEx) { + return backplaneUnavailable(fileId, backplaneEx); + } if (jobKey == null) { return ResponseEntity.notFound().build(); } + Optional> notOwner = guardNonOwner(jobKey); + if (notOwner.isPresent()) { + return notOwner.get(); + } + if (!validateJobAccess(jobKey)) { log.warn("Unauthorized attempt to download file: {}", fileId); return ResponseEntity.status(403) .body(Map.of("message", "You are not authorized to access this file")); } - // Retrieve file content - byte[] fileContent = fileStorage.retrieveBytes(fileId); - - // Find the file metadata from any job that contains this file - // This is for getting the original filename and content type ResultFile resultFile = taskManager.findResultFileByFileId(fileId); String fileName = resultFile != null ? resultFile.getFileName() : "download"; @@ -342,6 +338,8 @@ public class JobController { ? resultFile.getContentType() : MediaType.APPLICATION_OCTET_STREAM_VALUE; + byte[] fileContent = fileStorage.retrieveBytes(fileId); + return ResponseEntity.ok() .header("Content-Type", contentType) .header("Content-Disposition", createContentDispositionHeader(fileName)) @@ -357,11 +355,88 @@ public class JobController { } /** - * Create Content-Disposition header with UTF-8 filename support - * - * @param fileName The filename to encode - * @return Content-Disposition header value + * Returns 410 Gone when the job is owned by a peer node, empty otherwise. Uses a short-TTL + * local cache to avoid repeated Valkey lookups on the hot download path. When the backplane is + * unreachable, a locally-held job is still served and anything else gets a retryable 503. */ + private Optional> guardNonOwner(String jobId) { + if (clusterBackplane == null || jobStore == null) { + return Optional.empty(); + } + Optional entry; + Optional> cached = ownershipCache.get(jobId); + if (cached.isPresent()) { + entry = cached.get(); + } else { + try { + entry = jobStore.get(jobId); + } catch (RuntimeException ex) { + // Backplane unreachable: if we hold the job locally serve it, otherwise return a + // retryable 503 (same contract as the file endpoints) instead of a misleading 404. + if (taskManager.getJobResult(jobId) == null) { + return Optional.of(backplaneUnavailable(jobId, ex)); + } + log.warn( + "JobStore lookup failed for jobId={}; serving locally-held job: {}", + jobId, + ex.getMessage()); + return Optional.empty(); + } + ownershipCache.put(jobId, entry); + } + if (entry.isEmpty()) { + return Optional.empty(); + } + String owner = entry.get().owningNodeId(); + if (owner == null || owner.isBlank()) { + return Optional.empty(); + } + String localId = clusterBackplane.localNodeId(); + if (owner.equals(localId)) { + return Optional.empty(); + } + log.info( + "Sticky-session miss for jobId={} (owner={}, local={}); returning 410 so client" + + " retries via LB affinity", + jobId, + owner, + localId); + if (stickyMissRecorder != null) { + stickyMissRecorder.recordStickyMiss(); + } + return Optional.of( + ResponseEntity.status(410) + .header("Retry-After", "0") + .body( + Map.of( + "message", + "Result lives on another node. Retry to be routed there" + + " by the load balancer's sticky-session" + + " affinity, or re-run the job.", + "ownedBy", + owner, + "currentNode", + localId == null ? "" : localId))); + } + + /** + * When the backplane is unreachable we cannot resolve ownership or existence, and serving + * without that check would be unsafe - so return a retryable 503 (consistent with the + * sticky-410 retry model) rather than a misleading 404 or a generic 500. + */ + private ResponseEntity backplaneUnavailable(String id, RuntimeException ex) { + log.warn( + "Backplane lookup failed for {}; returning 503 (retryable): {}", + id, + ex.getMessage()); + return ResponseEntity.status(503) + .header("Retry-After", "1") + .body( + Map.of( + "message", + "Cluster backplane temporarily unavailable; retry shortly.")); + } + private String createContentDispositionHeader(String fileName) { try { String encodedFileName = @@ -371,19 +446,11 @@ public class JobController { .replaceAll("%20"); // URLEncoder uses + for spaces, but we want %20 return "attachment; filename=\"" + fileName + "\"; filename*=UTF-8''" + encodedFileName; } catch (Exception e) { - // Fallback to basic filename if encoding fails return "attachment; filename=\"" + fileName + "\""; } } - /** - * Validate that the current user has access to the given job. - * - * @param jobId the job identifier to validate - * @return true if user has access, false otherwise - */ private boolean validateJobAccess(String jobId) { - // If JobOwnershipService is available (security enabled), use it if (jobOwnershipService != null) { try { return jobOwnershipService.validateJobAccess(jobId); @@ -393,8 +460,6 @@ public class JobController { } } - // Security disabled - allow all access (backwards compatibility) - // When security is not enabled, any user can access any job by jobId return true; } } diff --git a/app/core/src/main/java/stirling/software/common/controller/JobOwnershipCache.java b/app/core/src/main/java/stirling/software/common/controller/JobOwnershipCache.java new file mode 100644 index 000000000..fd89a16ab --- /dev/null +++ b/app/core/src/main/java/stirling/software/common/controller/JobOwnershipCache.java @@ -0,0 +1,47 @@ +package stirling.software.common.controller; + +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import stirling.software.common.cluster.JobStoreEntry; + +/** + * Process-local TTL cache for {@link JobStoreEntry} lookups to suppress redundant Valkey HGETALL + * round-trips on the hot result-download path (sticky-410 ownership check). + * + *

5 second TTL is short enough that a job's lifecycle transitions (RUNNING -> COMPLETE -> TTL + * expiry) propagate to all nodes within the LB's sticky-session window, and short enough that a + * mistakenly-cached "not found" recovers quickly when an entry actually shows up. Cap the map at + * 2048 entries to bound memory; eviction is best-effort (clear-and-restart) since the cache is + * advisory. + */ +final class JobOwnershipCache { + + private static final long TTL_NANOS = 5L * 1_000_000_000L; // 5 s + private static final int MAX_ENTRIES = 2048; + + private final ConcurrentMap entries = new ConcurrentHashMap<>(); + + Optional> get(String jobId) { + Entry e = entries.get(jobId); + if (e == null) { + return Optional.empty(); + } + if (System.nanoTime() - e.storedAtNanos > TTL_NANOS) { + entries.remove(jobId, e); + return Optional.empty(); + } + return Optional.of(e.value); + } + + void put(String jobId, Optional value) { + if (entries.size() >= MAX_ENTRIES) { + // Best-effort eviction; under burst the cache simply rebuilds. + entries.clear(); + } + entries.put(jobId, new Entry(value, System.nanoTime())); + } + + private record Entry(Optional value, long storedAtNanos) {} +} diff --git a/app/core/src/test/java/stirling/software/common/controller/JobControllerOwnershipTest.java b/app/core/src/test/java/stirling/software/common/controller/JobControllerOwnershipTest.java new file mode 100644 index 000000000..040f72920 --- /dev/null +++ b/app/core/src/test/java/stirling/software/common/controller/JobControllerOwnershipTest.java @@ -0,0 +1,479 @@ +package stirling.software.common.controller; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.util.ReflectionTestUtils; + +import jakarta.servlet.http.HttpServletRequest; + +import stirling.software.common.cluster.ClusterBackplane; +import stirling.software.common.cluster.JobStore; +import stirling.software.common.cluster.JobStoreEntry; +import stirling.software.common.cluster.StickyMissRecorder; +import stirling.software.common.model.job.JobResult; +import stirling.software.common.service.FileStorage; +import stirling.software.common.service.JobOwnershipService; +import stirling.software.common.service.JobQueue; +import stirling.software.common.service.TaskManager; + +/** + * Sticky-410 ownership contract for {@link JobController}: peer-owned jobs return 410 Gone with + * ownedBy/currentNode fields; locally-owned and no-entry cases return 200. FileStorage is never + * touched on the 410 path. Manual mock construction so tests can vary backplane/jobstore combos. + */ +class JobControllerOwnershipTest { + + private TaskManager taskManager; + private FileStorage fileStorage; + private JobQueue jobQueue; + private HttpServletRequest request; + private JobOwnershipService jobOwnershipService; + private ClusterBackplane clusterBackplane; + private JobStore jobStore; + private StickyMissRecorder stickyMissRecorder; + + private static final String JOB_ID = "job-42"; + private static final String FILE_ID = "file-abc"; + private static final String LOCAL_NODE = "node-self"; + private static final String PEER_NODE = "node-peer"; + + @BeforeEach + void setUp() { + taskManager = mock(TaskManager.class); + fileStorage = mock(FileStorage.class); + jobQueue = mock(JobQueue.class); + request = mock(HttpServletRequest.class); + jobOwnershipService = mock(JobOwnershipService.class); + clusterBackplane = mock(ClusterBackplane.class); + jobStore = mock(JobStore.class); + stickyMissRecorder = mock(StickyMissRecorder.class); + } + + private JobController makeController(ClusterBackplane backplane, JobStore store) { + JobController c = + new JobController(taskManager, fileStorage, jobQueue, request, backplane, store); + ReflectionTestUtils.setField(c, "stickyMissRecorder", stickyMissRecorder); + return c; + } + + private JobController makeController() { + return makeController(clusterBackplane, jobStore); + } + + private JobStoreEntry entryOwnedBy(String ownerNodeId) { + return new JobStoreEntry( + JOB_ID, + JobStoreEntry.JobState.COMPLETE, + ownerNodeId, + Instant.now(), + Instant.now(), + null, + List.of(FILE_ID), + Map.of()); + } + + private JobResult completedJobWithFile() { + JobResult result = new JobResult(); + result.setJobId(JOB_ID); + // completeWithSingleFile populates the resultFiles list, sets complete=true, + // and sets completedAt - all required for the getJobResult single-file branch. + result.completeWithSingleFile(FILE_ID, "out.pdf", "application/pdf", 7L); + return result; + } + + @Test + @DisplayName( + "downloadFile peer-owned → full sticky-410 contract" + + " (status + Retry-After + payload + metric + storage untouched)") + void downloadFile_peerOwned_fullStickyContract() throws Exception { + when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE); + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE))); + + ResponseEntity response = makeController().downloadFile(FILE_ID); + + assertEquals(HttpStatus.GONE, response.getStatusCode()); + assertEquals("0", response.getHeaders().getFirst("Retry-After")); + + assertInstanceOf(Map.class, response.getBody()); + Map body = (Map) response.getBody(); + assertEquals(3, body.size(), "exactly: message, ownedBy, currentNode"); + assertEquals(PEER_NODE, body.get("ownedBy")); + assertEquals(LOCAL_NODE, body.get("currentNode")); + assertNotNull(body.get("message")); + assertTrue(((String) body.get("message")).toLowerCase().contains("retry")); + assertNull(body.get("internalSecret")); + assertNull(body.get("filePath")); + verify(stickyMissRecorder).recordStickyMiss(); + verify(fileStorage, never()).retrieveBytes(FILE_ID); + } + + private static Stream downloadHappyPathScenarios() { + return Stream.of( + Arguments.of("locallyOwned", LOCAL_NODE, true), + Arguments.of("noJobStoreEntry", null, false), + Arguments.of("blankOwningNodeId", "", true)); + } + + @ParameterizedTest(name = "downloadFile {0} -> 200, no sticky-miss") + @MethodSource("downloadHappyPathScenarios") + void downloadFile_happyPath_returnsOkAndNoMetric( + String scenario, String ownerNodeId, boolean entryPresent) throws Exception { + when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE); + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + when(jobStore.get(JOB_ID)) + .thenReturn( + entryPresent ? Optional.of(entryOwnedBy(ownerNodeId)) : Optional.empty()); + when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes()); + + ResponseEntity response = makeController().downloadFile(FILE_ID); + + assertEquals(HttpStatus.OK, response.getStatusCode(), scenario); + verify(fileStorage).retrieveBytes(FILE_ID); + verify(stickyMissRecorder, never()).recordStickyMiss(); + } + + @Test + @DisplayName("getJobResult: locally-owned single-file result → reads from FileStorage, 200 OK") + void getJobResult_singleFile_locallyOwned_readsFromStorage() throws Exception { + when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE); + when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile()); + when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE))); + when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes()); + + ResponseEntity response = makeController().getJobResult(JOB_ID); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + } + + private enum Endpoint { + DOWNLOAD_FILE, + GET_JOB_RESULT, + GET_JOB_STATUS, + GET_JOB_FILES, + GET_FILE_METADATA, + CANCEL_JOB + } + + private static Stream peerOwned410Scenarios() { + return Stream.of( + Arguments.of(Endpoint.DOWNLOAD_FILE), + Arguments.of(Endpoint.GET_JOB_RESULT), + Arguments.of(Endpoint.GET_JOB_STATUS), + Arguments.of(Endpoint.GET_JOB_FILES), + Arguments.of(Endpoint.GET_FILE_METADATA), + Arguments.of(Endpoint.CANCEL_JOB)); + } + + @ParameterizedTest(name = "{0} peer-owned -> 410, ownedBy=peer, metric++") + @MethodSource("peerOwned410Scenarios") + void endpoint_peerOwned_returns410(Endpoint endpoint) throws Exception { + when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE); + when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE))); + switch (endpoint) { + case DOWNLOAD_FILE, GET_FILE_METADATA -> + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + case GET_JOB_RESULT -> + when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile()); + case GET_JOB_STATUS, GET_JOB_FILES -> + when(taskManager.getJobResult(JOB_ID)).thenReturn(null); + case CANCEL_JOB -> { + when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false); + when(taskManager.getJobResult(JOB_ID)).thenReturn(null); + } + } + + ResponseEntity response = + switch (endpoint) { + case DOWNLOAD_FILE -> makeController().downloadFile(FILE_ID); + case GET_JOB_RESULT -> makeController().getJobResult(JOB_ID); + case GET_JOB_STATUS -> makeController().getJobStatus(JOB_ID); + case GET_JOB_FILES -> makeController().getJobFiles(JOB_ID); + case GET_FILE_METADATA -> makeController().getFileMetadata(FILE_ID); + case CANCEL_JOB -> makeController().cancelJob(JOB_ID); + }; + + assertEquals(HttpStatus.GONE, response.getStatusCode()); + Map body = (Map) response.getBody(); + assertEquals(PEER_NODE, body.get("ownedBy")); + assertEquals(LOCAL_NODE, body.get("currentNode")); + verify(stickyMissRecorder).recordStickyMiss(); + verify(fileStorage, never()).retrieveBytes(FILE_ID); + if (endpoint == Endpoint.CANCEL_JOB) { + verify(taskManager, never()).setError(JOB_ID, "Job was cancelled by user"); + } + } + + private static Stream unknownJob404Scenarios() { + return Stream.of(Arguments.of(Endpoint.GET_JOB_STATUS), Arguments.of(Endpoint.CANCEL_JOB)); + } + + @ParameterizedTest(name = "{0} unknown jobId -> 404 (not 410), no metric") + @MethodSource("unknownJob404Scenarios") + void endpoint_unknownJob_returns404(Endpoint endpoint) { + when(taskManager.getJobResult(JOB_ID)).thenReturn(null); + when(jobStore.get(JOB_ID)).thenReturn(Optional.empty()); + if (endpoint == Endpoint.CANCEL_JOB) { + when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false); + } + + ResponseEntity response = + switch (endpoint) { + case GET_JOB_STATUS -> makeController().getJobStatus(JOB_ID); + case CANCEL_JOB -> makeController().cancelJob(JOB_ID); + default -> throw new IllegalArgumentException(endpoint.name()); + }; + + assertEquals(HttpStatus.NOT_FOUND, response.getStatusCode()); + verify(stickyMissRecorder, never()).recordStickyMiss(); + } + + @Test + @DisplayName("Single-instance install (no ClusterBackplane bean): no 410, no NPE") + void singleInstance_noClusterBackplane_noGoneResponse() throws Exception { + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes()); + + ResponseEntity response = makeController(null, jobStore).downloadFile(FILE_ID); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + verify(fileStorage).retrieveBytes(FILE_ID); + } + + @Test + @DisplayName("Single-instance install (no JobStore bean): no 410, no NPE") + void singleInstance_noJobStore_noGoneResponse() throws Exception { + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes()); + + ResponseEntity response = makeController(clusterBackplane, null).downloadFile(FILE_ID); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + verify(fileStorage).retrieveBytes(FILE_ID); + } + + @Test + @DisplayName("Single-instance (no StickyMissRecorder bean) → no NPE, still 200 OK") + void noStickyMissRecorder_works() throws Exception { + when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE); + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE))); + when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes()); + + JobController c = makeController(); + ReflectionTestUtils.setField(c, "stickyMissRecorder", null); + + ResponseEntity response = c.downloadFile(FILE_ID); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + } + + @Test + @DisplayName( + "cluster-mode but localNodeId is null → no NPE; 410 because owner is set and" + + " differs from blank") + void clusterBackplanePresent_butLocalNodeIdNull_falsBackGracefully() throws Exception { + when(clusterBackplane.localNodeId()).thenReturn(null); + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE))); + + // We still 410: owner is "node-peer", local is null → they don't match. Rather than + // silently 200-from-wrong-disk (which would serve garbage), we surface the mismatch. + ResponseEntity response = makeController().downloadFile(FILE_ID); + + assertEquals(HttpStatus.GONE, response.getStatusCode()); + Map body = (Map) response.getBody(); + assertEquals("", body.get("currentNode"), "blank when localNodeId is null"); + assertEquals(PEER_NODE, body.get("ownedBy")); + } + + @Test + @DisplayName("Owner returns 410 even when JobOwnershipService allows access (orthogonal)") + void ownershipService_passes_butStickyStillReturns410() throws Exception { + when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE); + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE))); + lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(true); + + JobController c = makeController(); + ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService); + ResponseEntity response = c.downloadFile(FILE_ID); + + assertEquals(HttpStatus.GONE, response.getStatusCode()); + } + + @Test + @DisplayName( + "downloadFile: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak" + + " file existence") + void downloadFile_peerOwned_ownershipDenied_returns410NotForbidden() throws Exception { + when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE); + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE))); + lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false); + + JobController c = makeController(); + ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService); + ResponseEntity response = c.downloadFile(FILE_ID); + + assertEquals(HttpStatus.GONE, response.getStatusCode()); + Map body = (Map) response.getBody(); + assertEquals(PEER_NODE, body.get("ownedBy")); + verify(fileStorage, never()).retrieveBytes(FILE_ID); + } + + @Test + @DisplayName( + "getJobStatus: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak" + + " job existence") + void getJobStatus_peerOwned_ownershipDenied_returns410NotForbidden() { + when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE); + when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE))); + lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false); + + JobController c = makeController(); + ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService); + ResponseEntity response = c.getJobStatus(JOB_ID); + + assertEquals(HttpStatus.GONE, response.getStatusCode()); + Map body = (Map) response.getBody(); + assertEquals(PEER_NODE, body.get("ownedBy")); + } + + @Test + @DisplayName( + "cancelJob: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak job" + + " existence") + void cancelJob_peerOwned_ownershipDenied_returns410NotForbidden() { + when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE); + when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false); + when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE))); + lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false); + + JobController c = makeController(); + ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService); + ResponseEntity response = c.cancelJob(JOB_ID); + + assertEquals(HttpStatus.GONE, response.getStatusCode()); + Map body = (Map) response.getBody(); + assertEquals(PEER_NODE, body.get("ownedBy")); + verify(taskManager, never()).setError(JOB_ID, "Job was cancelled by user"); + } + + @Test + @DisplayName( + "guardNonOwner caches JobStore.get within TTL window: second call same jobId hits" + + " cache, not Valkey") + void guardNonOwner_cachesJobStoreLookupWithinTtl() throws Exception { + when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE); + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE))); + when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes()); + + JobController c = makeController(); + c.downloadFile(FILE_ID); + c.downloadFile(FILE_ID); + c.downloadFile(FILE_ID); + + verify(jobStore, times(1)).get(JOB_ID); + } + + @Test + @DisplayName( + "guardNonOwner: JobStore.get throws (Valkey timeout) → falls through to local-disk" + + " path, no 500 leaks to caller") + void guardNonOwner_jobStoreException_fallsThroughToLocalPath() throws Exception { + when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID); + when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout")); + when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile()); + when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes()); + + ResponseEntity response = makeController().downloadFile(FILE_ID); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + verify(fileStorage).retrieveBytes(FILE_ID); + verify(stickyMissRecorder, never()).recordStickyMiss(); + } + + @Test + @DisplayName("backplane down + job NOT held locally → 503 retryable (not a misleading 404)") + void jobEndpoint_backplaneDown_notLocal_returns503() { + when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout")); + when(taskManager.getJobResult(JOB_ID)).thenReturn(null); + + ResponseEntity response = makeController().getJobStatus(JOB_ID); + + assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode()); + assertEquals("1", response.getHeaders().getFirst("Retry-After")); + verify(stickyMissRecorder, never()).recordStickyMiss(); + } + + @Test + @DisplayName("backplane down but job held locally → owner still serves (not 503)") + void jobEndpoint_backplaneDown_local_servesLocally() { + when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout")); + when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile()); + + ResponseEntity response = makeController().getJobStatus(JOB_ID); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + } + + @Test + @DisplayName( + "downloadFile: findJobKeyByFileId throws (backplane down) → 503 + Retry-After," + + " not 404/500, storage untouched") + void downloadFile_findJobKeyThrows_returns503Retryable() throws Exception { + when(taskManager.findJobKeyByFileId(FILE_ID)) + .thenThrow(new RuntimeException("Valkey command timeout")); + + ResponseEntity response = makeController().downloadFile(FILE_ID); + + assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode()); + assertEquals("1", response.getHeaders().getFirst("Retry-After")); + Map body = (Map) response.getBody(); + assertTrue(((String) body.get("message")).toLowerCase().contains("unavailable")); + verify(fileStorage, never()).retrieveBytes(FILE_ID); + } + + @Test + @DisplayName( + "getFileMetadata: findJobKeyByFileId throws (backplane down) → 503 + Retry-After," + + " not 404/500") + void getFileMetadata_findJobKeyThrows_returns503Retryable() throws Exception { + when(taskManager.findJobKeyByFileId(FILE_ID)) + .thenThrow(new RuntimeException("Valkey command timeout")); + + ResponseEntity response = makeController().getFileMetadata(FILE_ID); + + assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode()); + assertEquals("1", response.getHeaders().getFirst("Retry-After")); + Map body = (Map) response.getBody(); + assertTrue(((String) body.get("message")).toLowerCase().contains("unavailable")); + verify(fileStorage, never()).retrieveBytes(FILE_ID); + } +} diff --git a/app/core/src/test/java/stirling/software/common/controller/JobControllerTest.java b/app/core/src/test/java/stirling/software/common/controller/JobControllerTest.java index 1388abeae..53212a2e6 100644 --- a/app/core/src/test/java/stirling/software/common/controller/JobControllerTest.java +++ b/app/core/src/test/java/stirling/software/common/controller/JobControllerTest.java @@ -19,6 +19,8 @@ import org.springframework.test.util.ReflectionTestUtils; import jakarta.servlet.http.HttpServletRequest; +import stirling.software.common.cluster.ClusterBackplane; +import stirling.software.common.cluster.JobStore; import stirling.software.common.model.job.JobResult; import stirling.software.common.service.FileStorage; import stirling.software.common.service.JobOwnershipService; @@ -37,6 +39,10 @@ class JobControllerTest { @Mock private JobOwnershipService jobOwnershipService; + @Mock private ClusterBackplane clusterBackplane; + + @Mock private JobStore jobStore; + private MockHttpSession session; @InjectMocks private JobController controller; diff --git a/app/proprietary/build.gradle b/app/proprietary/build.gradle index 4c9932d99..6f744c6c6 100644 --- a/app/proprietary/build.gradle +++ b/app/proprietary/build.gradle @@ -55,8 +55,13 @@ dependencies { api 'org.springframework.boot:spring-boot-starter-mail' api 'org.springframework.boot:spring-boot-starter-cache' api 'com.github.ben-manes.caffeine:caffeine' + implementation 'org.springframework.boot:spring-boot-starter-data-redis' api 'io.swagger.core.v3:swagger-core-jakarta:2.2.46' - implementation 'com.bucket4j:bucket4j_jdk17-core:8.18.0' + implementation 'com.bucket4j:bucket4j_jdk17-core:8.19.0' + // Lettuce-backed Bucket4j ProxyManager used by ValkeyRateLimitStore for cluster-wide + // token-bucket rate limiting (parity with in-process Bucket4j semantics; no fixed-window + // boundary doubling). + implementation 'com.bucket4j:bucket4j_jdk17-lettuce:8.19.0' // https://mvnrepository.com/artifact/com.bucket4j/bucket4j_jdk17 implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion" @@ -77,9 +82,12 @@ dependencies { implementation "software.amazon.awssdk:s3:$awsSdkVersion" implementation "software.amazon.awssdk:url-connection-client:$awsSdkVersion" + // Testcontainers: real MinIO/LocalStack (S3) and Valkey for integration tests in CI without + // manually-started instances. Tests skip cleanly when Docker is unavailable. + testImplementation "org.testcontainers:testcontainers:$testcontainersMinioVersion" testImplementation "org.testcontainers:minio:$testcontainersMinioVersion" - testImplementation "org.testcontainers:junit-jupiter:$testcontainersMinioVersion" testImplementation "org.testcontainers:localstack:$testcontainersMinioVersion" + testImplementation "org.testcontainers:junit-jupiter:$testcontainersMinioVersion" } tasks.register('prepareKotlinBuildScriptModel') {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterLicenseGate.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterLicenseGate.java new file mode 100644 index 000000000..cc67354fe --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterLicenseGate.java @@ -0,0 +1,40 @@ +package stirling.software.proprietary.cluster; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Configuration; + +import jakarta.annotation.PostConstruct; + +import lombok.extern.slf4j.Slf4j; + +/** + * Runtime license gate for cluster mode. Cluster mode requires a SERVER or ENTERPRISE license; the + * SaaS flavor bypasses (no {@code runningProOrHigher} bean is published). The Valkey connection + * config {@code @DependsOn} this bean, so it runs before any Valkey bean is constructed. + */ +@Configuration +@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true") +@Slf4j +public class ClusterLicenseGate { + + @Autowired(required = false) + @Qualifier("runningProOrHigher") + private Boolean runningProOrHigher; + + @PostConstruct + void verifyLicense() { + if (runningProOrHigher == null) { + return; // saas flavor - licensed via Stripe elsewhere + } + if (!runningProOrHigher) { + throw new IllegalStateException( + "Cluster mode (cluster.enabled=true) requires a SERVER or" + + " ENTERPRISE license. Configure stirling.premium.key with a valid" + + " license key (contact sales@stirlingpdf.com to obtain one), or set" + + " cluster.enabled=false."); + } + log.info("Cluster license gate: SERVER/ENTERPRISE license verified, cluster mode allowed."); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterMetrics.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterMetrics.java new file mode 100644 index 000000000..a97c3bb90 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterMetrics.java @@ -0,0 +1,113 @@ +package stirling.software.proprietary.cluster; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; + +import stirling.software.common.cluster.StickyMissRecorder; +import stirling.software.common.model.ApplicationProperties; + +/** + * Cluster operation metrics exposed via {@code /actuator/prometheus}. Registered only when cluster + * mode is on. + */ +@Component +@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true") +public class ClusterMetrics implements StickyMissRecorder { + + private final MeterRegistry registry; + private final ApplicationProperties applicationProperties; + + private final Counter stickyMissTotal; + private final Counter rateLimitRejected; + private final Timer backplaneLatency; + private final Timer jobWaitSeconds; + + // Per-lane queue depth gauges. Lanes are a fixed enum (FAST, SLOW, AI), so we register all + // three eagerly so dashboards never have a missing series. + private static final List KNOWN_LANES = List.of("FAST", "SLOW", "AI"); + private final ConcurrentHashMap queueDepth = new ConcurrentHashMap<>(); + + private final AtomicLong jobsInflight = new AtomicLong(); + + public ClusterMetrics(MeterRegistry registry, ApplicationProperties applicationProperties) { + this.registry = registry; + this.applicationProperties = applicationProperties; + this.stickyMissTotal = + Counter.builder("stirling_cluster_sticky_miss_total") + .description( + "Sticky-session misses: a download for a job whose result lives on" + + " a peer node landed on this node. High sustained value means" + + " LB affinity is broken.") + .register(registry); + this.rateLimitRejected = + Counter.builder("stirling_cluster_ratelimit_rejected_total") + .description("Cluster-wide rate limit rejections") + .register(registry); + this.backplaneLatency = + Timer.builder("stirling_cluster_backplane_latency_seconds") + .description("Backplane round-trip latency") + .register(registry); + this.jobWaitSeconds = + Timer.builder("stirling_cluster_job_wait_seconds") + .description("Time jobs spend queued before execution") + .register(registry); + Gauge.builder("stirling_cluster_jobs_inflight", jobsInflight, AtomicLong::doubleValue) + .description("Jobs currently in flight on this node") + .tag("node", applicationProperties.getCluster().resolvedNodeId()) + .register(registry); + for (String lane : KNOWN_LANES) { + ensureLaneGauge(lane); + } + } + + @Override + public void recordStickyMiss() { + stickyMissTotal.increment(); + } + + public void recordRateLimitReject() { + rateLimitRejected.increment(); + } + + public Timer backplaneLatency() { + return backplaneLatency; + } + + public Timer jobWaitSeconds() { + return jobWaitSeconds; + } + + public void incrementInflight() { + jobsInflight.incrementAndGet(); + } + + public void decrementInflight() { + jobsInflight.decrementAndGet(); + } + + public void setQueueDepth(String lane, long depth) { + ensureLaneGauge(lane).set(depth); + } + + private AtomicLong ensureLaneGauge(String lane) { + return queueDepth.computeIfAbsent( + lane, + l -> { + AtomicLong holder = new AtomicLong(); + Gauge.builder("stirling_cluster_queue_depth", holder, AtomicLong::doubleValue) + .description("Pending items in a job queue lane") + .tag("lane", l) + .register(registry); + return holder; + }); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterNodeBootstrap.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterNodeBootstrap.java new file mode 100644 index 000000000..ccd399410 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterNodeBootstrap.java @@ -0,0 +1,201 @@ +package stirling.software.proprietary.cluster; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.time.Duration; +import java.time.Instant; +import java.util.Locale; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.SmartLifecycle; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.cluster.ClusterNode; +import stirling.software.common.cluster.InstanceRegistry; +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.model.ApplicationProperties.Cluster; + +/** + * Registers the local node with {@link InstanceRegistry} on startup, refreshes the entry at 1/3 of + * the TTL, and deregisters cleanly on shutdown. + * + *

Implements {@link SmartLifecycle} with {@code getPhase() == Integer.MAX_VALUE} so Spring tears + * this bean down before {@code LettuceConnectionFactory} - deregister therefore runs while the + * Valkey connection is still alive. + */ +@Component +@Slf4j +@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true") +public class ClusterNodeBootstrap implements SmartLifecycle { + + private final Duration heartbeatTtl; + + private final ApplicationProperties applicationProperties; + private final InstanceRegistry instanceRegistry; + + @Value("${server.port:8080}") + private int serverPort; + + private volatile String nodeId; + private volatile String internalAddress; + private volatile boolean running = false; + + public ClusterNodeBootstrap( + ApplicationProperties applicationProperties, InstanceRegistry instanceRegistry) { + this.applicationProperties = applicationProperties; + this.instanceRegistry = instanceRegistry; + Cluster cluster = applicationProperties.getCluster(); + // Default must match the @Scheduled fallback below AND the model default + // (ApplicationProperties.Cluster.Node.heartbeatIntervalMs = 5000); otherwise the TTL is + // computed from a different interval than the scheduler runs at and the 3x margin breaks. + long heartbeatMs = + cluster.getNode() == null ? 5000L : cluster.getNode().getHeartbeatIntervalMs(); + // TTL = 3x heartbeat: tolerate two missed ticks before the node drops out of the registry. + this.heartbeatTtl = Duration.ofMillis(heartbeatMs * 3); + } + + @EventListener(ApplicationReadyEvent.class) + public void registerOnStartup() { + nodeId = applicationProperties.getCluster().resolvedNodeId(); + internalAddress = resolveInternalAddress(); + registerSelf("register"); + } + + @Scheduled(fixedDelayString = "${cluster.node.heartbeat-interval-ms:5000}") + public void heartbeat() { + // Heartbeat-after-stop race: SmartLifecycle.stop() deregisters, but the @Scheduled + // tick keeps firing during a slow drain. Without this guard, the next tick re-registers + // the dead node and the entry resurfaces in the registry until TTL expiry. + if (!running) { + return; + } + if (nodeId == null) { + return; // not yet registered (startup race) + } + // Self-healing: register() is idempotent and re-populates every field, so a wiped + // Valkey (FLUSHALL, hash eviction) recovers on the next tick without operator action. + registerSelf("heartbeat"); + } + + private void registerSelf(String reason) { + try { + instanceRegistry.register( + new ClusterNode(nodeId, internalAddress, Instant.now(), role()), heartbeatTtl); + if ("register".equals(reason)) { + log.info( + "Cluster node registered: nodeId={}, internalAddress={}, role={}, ttl={}s", + nodeId, + internalAddress, + role(), + heartbeatTtl.toSeconds()); + } + } catch (RuntimeException e) { + log.debug("Cluster {} failed for {}", reason, nodeId, e); + } + } + + @Override + public void start() { + running = true; + } + + @Override + public void stop() { + running = false; + if (nodeId == null) { + return; + } + try { + instanceRegistry.deregister(nodeId); + log.info("Cluster node deregistered: {}", nodeId); + } catch (RuntimeException e) { + // Registry entry will TTL-expire within heartbeatTtl anyway. + log.warn( + "Cluster deregister failed for {} (will TTL-expire within {}s): {}", + nodeId, + heartbeatTtl.toSeconds(), + e.getMessage()); + } + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public int getPhase() { + return Integer.MAX_VALUE; + } + + @Override + public boolean isAutoStartup() { + return true; + } + + /** + * Resolve the address peers should hit. Order: explicit config -> {@code POD_IP} env (K8s + * downward API) -> JDK hostname -> fail loud (never silently fall back to a loopback). + * + *

Scheme is taken from {@code cluster.node.scheme} (default {@code http}). Set to {@code + * https} when nodes terminate TLS themselves; leave as {@code http} when an upstream LB + * terminates TLS and intra-cluster traffic is plain HTTP. + */ + private String resolveInternalAddress() { + Cluster cluster = applicationProperties.getCluster(); + String configured = + cluster.getNode() == null ? null : cluster.getNode().getInternalAddress(); + if (configured != null && !configured.isBlank()) { + return ensurePort(configured); + } + String podIp = System.getenv("POD_IP"); + if (podIp != null && !podIp.isBlank()) { + return scheme() + "://" + podIp + ":" + serverPort; + } + try { + return scheme() + + "://" + + InetAddress.getLocalHost().getHostAddress() + + ":" + + serverPort; + } catch (UnknownHostException e) { + throw new IllegalStateException( + "Could not resolve this host's address for cluster registration; set" + + " cluster.node.internal-address explicitly (or set POD_IP" + + " in the Kubernetes downward API).", + e); + } + } + + private String ensurePort(String addr) { + if (addr.startsWith("http://") || addr.startsWith("https://")) { + return addr; + } + if (addr.contains(":")) { + return scheme() + "://" + addr; + } + return scheme() + "://" + addr + ":" + serverPort; + } + + private String scheme() { + Cluster cluster = applicationProperties.getCluster(); + if (cluster.getNode() == null + || cluster.getNode().getScheme() == null + || cluster.getNode().getScheme().isBlank()) { + return "http"; + } + String s = cluster.getNode().getScheme().trim().toLowerCase(Locale.ROOT); + return "https".equals(s) ? "https" : "http"; + } + + private String role() { + Cluster.NodeRole r = applicationProperties.getCluster().resolvedRole(); + return r == null ? "BOTH" : r.name(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ConditionalOnValkeyBackplane.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ConditionalOnValkeyBackplane.java new file mode 100644 index 000000000..4651cf6f2 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ConditionalOnValkeyBackplane.java @@ -0,0 +1,19 @@ +package stirling.software.proprietary.cluster.valkey; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +/** + * Composite condition: matches only when cluster.enabled=true AND cluster.backplane=valkey. Both + * checks are required (enabled alone may select the in-process backplane, which must not load + * Valkey beans); a single {@code @ConditionalOnExpression} keeps the guard in one place. + */ +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression( + "${cluster.enabled:false} and '${cluster.backplane:inprocess}'.equals('valkey')") +public @interface ConditionalOnValkeyBackplane {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplane.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplane.java new file mode 100644 index 000000000..cf7bfb61c --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplane.java @@ -0,0 +1,55 @@ +package stirling.software.proprietary.cluster.valkey; + +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.cluster.ClusterBackplane; +import stirling.software.common.model.ApplicationProperties; + +@Slf4j +@Component +@RequiredArgsConstructor +@ConditionalOnValkeyBackplane +public class ValkeyClusterBackplane implements ClusterBackplane { + + private final ApplicationProperties applicationProperties; + private final StringRedisTemplate template; + + @Override + public boolean isHealthy() { + try { + // template.execute() borrows from the pool and returns the connection in a finally + // block - critical because isHealthy() is hit on every k8s liveness/readiness probe + // tick. Calling getConnectionFactory().getConnection() directly leaks the connection + // and exhausts the pool under monitoring load. + String pong = template.execute((RedisCallback) connection -> connection.ping()); + return "PONG".equalsIgnoreCase(pong); + } catch (RuntimeException ex) { + log.warn("Valkey backplane health check failed: {}", ex.getMessage()); + return false; + } + } + + @Override + public String backplaneType() { + return "valkey"; + } + + @Override + public String localNodeId() { + return applicationProperties.getCluster().resolvedNodeId(); + } + + /** + * Valkey TTL evicts job entries; local cleanup loop is redundant and would race with cluster + * state. + */ + @Override + public boolean shouldRunLocalCleanup() { + return false; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfiguration.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfiguration.java new file mode 100644 index 000000000..ce121ec66 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfiguration.java @@ -0,0 +1,265 @@ +package stirling.software.proprietary.cluster.valkey; + +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Duration; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisPassword; +import org.springframework.data.redis.connection.RedisStandaloneConfiguration; +import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration; +import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.data.redis.core.StringRedisTemplate; + +import io.lettuce.core.RedisCommandExecutionException; +import io.lettuce.core.SslVerifyMode; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.model.ApplicationProperties.Cluster; + +@Slf4j +@Configuration +@RequiredArgsConstructor +@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true") +@DependsOn("clusterLicenseGate") +public class ValkeyConnectionConfiguration { + + private final ApplicationProperties applicationProperties; + + @Bean(destroyMethod = "destroy") + @ConditionalOnProperty(name = "cluster.backplane", havingValue = "valkey") + public LettuceConnectionFactory valkeyConnectionFactory() { + Cluster cluster = applicationProperties.getCluster(); + Endpoint endpoint = parseUrl(cluster.getValkey().getUrl()); + RedisStandaloneConfiguration cfg = + new RedisStandaloneConfiguration(endpoint.host(), endpoint.port()); + if (endpoint.username() != null) { + cfg.setUsername(endpoint.username()); + } + if (endpoint.password() != null) { + cfg.setPassword(RedisPassword.of(endpoint.password())); + } + boolean skipCertVerification = + cluster.getValkey().getTls() != null + && cluster.getValkey().getTls().isSkipCertVerification(); + LettuceClientConfiguration clientConfig = + buildClientConfiguration(endpoint.tls(), skipCertVerification); + LettuceConnectionFactory factory = new LettuceConnectionFactory(cfg, clientConfig); + factory.afterPropertiesSet(); + // Eager handshake with retry tolerates docker-compose DNS races; fails boot loudly + // if Valkey is genuinely unreachable. + eagerHandshake(factory, endpoint.host(), endpoint.port(), endpoint.tls()); + log.info( + "Valkey connection configured: {}:{} tls={} verifyPeer={}", + endpoint.host(), + endpoint.port(), + endpoint.tls(), + endpoint.tls() ? clientConfig.getVerifyMode() : "n/a"); + return factory; + } + + /** Parsed connection endpoint; username/password are null when absent. */ + record Endpoint(String host, int port, boolean tls, String username, String password) {} + + /** + * Parses {@code redis://[user:password@]host[:port]} (or {@code rediss://} for TLS) into an + * {@link Endpoint}. Package-private and side-effect-free so URL handling is unit-testable. + * + *

    + *
  • Missing port defaults to 6379. + *
  • {@code rediss} scheme selects TLS. + *
  • Userinfo {@code :password@} (empty user) is treated as password-only auth against the + * default user, not a login with an empty username. + *
  • Reserved characters in the password ({@code @ : / # ?}) must be percent-encoded; {@link + * URI} parses them structurally otherwise (e.g. {@code #} starts the fragment). + *
+ * + * @throws IllegalStateException if the URL is blank, syntactically invalid, or has no host + */ + static Endpoint parseUrl(String url) { + if (url == null || url.isBlank()) { + throw new IllegalStateException("cluster.valkey.url must be set when backplane=valkey"); + } + URI uri; + try { + uri = new URI(url); + } catch (URISyntaxException ex) { + throw new IllegalStateException( + "cluster.valkey.url is not a valid URI: " + url + " (" + ex.getMessage() + ")", + ex); + } + String host = uri.getHost(); + if (host == null || host.isBlank()) { + throw new IllegalStateException( + "cluster.valkey.url has no host: " + + url + + " (expected redis://[user:password@]host[:port])"); + } + boolean tls = "rediss".equalsIgnoreCase(uri.getScheme()); + int port = uri.getPort() <= 0 ? 6379 : uri.getPort(); + String username = null; + String password = null; + String userInfo = uri.getUserInfo(); + if (userInfo != null) { + String[] parts = userInfo.split(":", 2); + if (parts.length == 2) { + username = parts[0].isEmpty() ? null : parts[0]; + password = parts[1]; + } else if (!parts[0].isBlank()) { + password = parts[0]; + } + } + return new Endpoint(host, port, tls, username, password); + } + + /** + * Package-private for testing. verifyPeer(FULL) is pinned explicitly so a Spring Data Redis + * default change cannot silently weaken our TLS handshake. skipCertVerification is dev-only. + */ + static LettuceClientConfiguration buildClientConfiguration( + boolean tls, boolean skipCertVerification) { + LettuceClientConfiguration.LettuceClientConfigurationBuilder clientBuilder = + LettuceClientConfiguration.builder(); + // Bound every backplane command. Lettuce defaults to 60s; without this a partitioned or + // slow Valkey would stall hot-path calls (e.g. JobController.guardNonOwner -> jobStore.get + // on each request) for up to a minute, exhausting request threads. All backplane ops are + // non-blocking single commands, so a short timeout is safe. + clientBuilder.commandTimeout(Duration.ofSeconds(2)); + if (tls) { + clientBuilder + .useSsl() + .verifyPeer(skipCertVerification ? SslVerifyMode.NONE : SslVerifyMode.FULL); + if (skipCertVerification) { + log.warn( + "Valkey TLS hostname/chain verification DISABLED via" + + " cluster.valkey.tls.skip-cert-verification=true" + + " - insecure, dev-only"); + } + } + return clientBuilder.build(); + } + + /** + * 10 x 3s = 30s boot-time retry. Auth failures (WRONGPASS/NOAUTH/NOPERM) short-circuit + * immediately; only transport errors get the loop. Package-private for testing. + */ + static void eagerHandshake( + LettuceConnectionFactory factory, String host, int port, boolean tls) { + RuntimeException last = null; + for (int attempt = 1; attempt <= 10; attempt++) { + try { + String pong; + RedisConnection conn = factory.getConnection(); + try { + pong = conn.ping(); + } finally { + conn.close(); + } + if (!"PONG".equalsIgnoreCase(pong)) { + throw new IllegalStateException( + "Valkey PING returned '" + pong + "' (expected PONG)"); + } + if (attempt > 1) { + log.info("Valkey reachable after {} attempts", attempt); + } + return; + } catch (RuntimeException ex) { + if (isAuthFailure(ex)) { + factory.destroy(); + throw new IllegalStateException( + "Valkey authentication failed for " + + host + + ":" + + port + + " (tls=" + + tls + + "): " + + rootAuthMessage(ex) + + ". Check cluster.valkey.url credentials" + + " (user/password and ACL permissions).", + ex); + } + last = ex; + log.warn( + "Valkey PING attempt {}/10 failed ({}:{}, tls={}): {}", + attempt, + host, + port, + tls, + ex.getMessage()); + try { + Thread.sleep(3000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + } + } + factory.destroy(); + throw new IllegalStateException( + "Valkey unreachable at boot after 10 attempts (" + + host + + ":" + + port + + ", tls=" + + tls + + "): " + + (last == null ? "no detail" : last.getMessage()), + last); + } + + /** + * Walks the cause chain for WRONGPASS/NOAUTH/NOPERM replies. Spring Data Redis wraps Lettuce's + * RedisCommandExecutionException in RedisSystemException, so the auth signal may be one level + * down. No typed auth exception exists in spring-data-redis 4.0.5 / Lettuce 6.8.2. + */ + static boolean isAuthFailure(Throwable t) { + for (Throwable cur = t; cur != null; cur = cur.getCause()) { + if (cur instanceof RedisCommandExecutionException && hasAuthPrefix(cur.getMessage())) { + return true; + } + if (hasAuthPrefix(cur.getMessage())) { + return true; + } + if (cur.getCause() == cur) { + break; + } + } + return false; + } + + private static boolean hasAuthPrefix(String message) { + if (message == null) { + return false; + } + String upper = message.toUpperCase(java.util.Locale.ROOT).stripLeading(); + return upper.startsWith("WRONGPASS") + || upper.startsWith("NOAUTH") + || upper.startsWith("NOPERM"); + } + + private static String rootAuthMessage(Throwable t) { + for (Throwable cur = t; cur != null; cur = cur.getCause()) { + if (cur instanceof RedisCommandExecutionException && cur.getMessage() != null) { + return cur.getMessage(); + } + if (cur.getCause() == cur) { + break; + } + } + return t.getMessage(); + } + + @Bean + @ConditionalOnProperty(name = "cluster.backplane", havingValue = "valkey") + public StringRedisTemplate valkeyTemplate(LettuceConnectionFactory factory) { + return new StringRedisTemplate(factory); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyDistributedLock.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyDistributedLock.java new file mode 100644 index 000000000..0a70391c0 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyDistributedLock.java @@ -0,0 +1,102 @@ +package stirling.software.proprietary.cluster.valkey; + +import java.time.Duration; +import java.util.Collections; +import java.util.Optional; +import java.util.UUID; + +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; +import org.springframework.data.redis.core.script.RedisScript; +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.cluster.DistributedLock; + +@Component +@RequiredArgsConstructor +@ConditionalOnValkeyBackplane +@Slf4j +public class ValkeyDistributedLock implements DistributedLock { + + private static final String PREFIX = "stirling:lock:"; + + private static final RedisScript RELEASE_SCRIPT = + new DefaultRedisScript<>( + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", + Long.class); + + private static final RedisScript RENEW_SCRIPT = + new DefaultRedisScript<>( + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end", + Long.class); + + private final StringRedisTemplate template; + + @Override + public Optional tryAcquire(String lockKey, Duration leaseTime) { + String key = PREFIX + lockKey; + String value = UUID.randomUUID().toString(); + Boolean ok = template.opsForValue().setIfAbsent(key, value, leaseTime); + if (Boolean.TRUE.equals(ok)) { + return Optional.of(new ValkeyHandle(template, key, value)); + } + return Optional.empty(); + } + + private static final class ValkeyHandle implements LockHandle { + private final StringRedisTemplate template; + private final String key; + private final String value; + private boolean released; + + ValkeyHandle(StringRedisTemplate template, String key, String value) { + this.template = template; + this.key = key; + this.value = value; + } + + @Override + public synchronized void release() { + if (released) { + return; + } + released = true; + // Swallow + log: LockHandle is AutoCloseable, so release() runs from close() inside + // try-with-resources. An uncaught Valkey error here would mask the body's exception. + // The lease TTL-expires anyway, so a failed explicit release is safe. + try { + template.execute(RELEASE_SCRIPT, Collections.singletonList(key), value); + } catch (RuntimeException ex) { + log.warn( + "Lock release failed for {} (lease will TTL-expire): {}", + key, + ex.getMessage()); + } + } + + @Override + public synchronized boolean renew(Duration leaseTime) { + if (released) { + return false; + } + try { + Long result = + template.execute( + RENEW_SCRIPT, + Collections.singletonList(key), + value, + Long.toString(leaseTime.toMillis())); + return result != null && result == 1L; + } catch (RuntimeException ex) { + log.warn( + "Lock renew failed for {} (treated as lost lease): {}", + key, + ex.getMessage()); + return false; + } + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyInstanceRegistry.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyInstanceRegistry.java new file mode 100644 index 000000000..a8ff8d60e --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyInstanceRegistry.java @@ -0,0 +1,115 @@ +package stirling.software.proprietary.cluster.valkey; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.ScanOptions; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; + +import stirling.software.common.cluster.ClusterNode; +import stirling.software.common.cluster.InstanceRegistry; + +/** + * Valkey-backed {@link InstanceRegistry}. Each node is stored as a hash with a TTL equal to the + * configured heartbeat TTL; the heartbeat re-arms the TTL. + */ +@Component +@RequiredArgsConstructor +@ConditionalOnValkeyBackplane +public class ValkeyInstanceRegistry implements InstanceRegistry { + + private static final String PREFIX = "stirling:nodes:"; + + private final StringRedisTemplate template; + + @Override + public void register(ClusterNode node, Duration heartbeatTtl) { + String key = PREFIX + node.nodeId(); + long ttlMs = heartbeatTtl.toMillis(); + Map fields = new LinkedHashMap<>(); + fields.put("nodeId", node.nodeId()); + fields.put("internalAddress", node.internalAddress()); + fields.put("role", node.role()); + fields.put("lastHeartbeat", node.lastHeartbeat().toString()); + + // MULTI/EXEC so the hash fields and the TTL commit together. Without this, a crash + // between HSET and EXPIRE leaves the hash with no TTL: it never expires, masks the + // dead node as alive, and only a subsequent successful register() would re-arm it. + template.execute( + (RedisCallback) + connection -> { + connection.multi(); + byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8); + Map hashBytes = new LinkedHashMap<>(); + for (Map.Entry f : fields.entrySet()) { + hashBytes.put( + f.getKey().getBytes(StandardCharsets.UTF_8), + f.getValue().getBytes(StandardCharsets.UTF_8)); + } + connection.hashCommands().hMSet(keyBytes, hashBytes); + connection.keyCommands().pExpire(keyBytes, ttlMs); + connection.exec(); + return null; + }); + } + + @Override + public Optional lookup(String nodeId) { + return readNode(PREFIX + nodeId); + } + + @Override + public Collection activeNodes() { + ScanOptions options = ScanOptions.scanOptions().match(PREFIX + "*").count(256).build(); + List nodes = new ArrayList<>(); + try (Cursor cursor = template.scan(options)) { + while (cursor.hasNext()) { + readNode(cursor.next()).ifPresent(nodes::add); + } + } + return nodes; + } + + @Override + public void deregister(String nodeId) { + template.delete(PREFIX + nodeId); + } + + private Optional readNode(String key) { + Map entries = template.opsForHash().entries(key); + if (entries == null || entries.isEmpty()) { + return Optional.empty(); + } + Object nodeId = entries.get("nodeId"); + if (nodeId == null) { + return Optional.empty(); + } + Instant heartbeat = Instant.now(); + Object hb = entries.get("lastHeartbeat"); + if (hb != null) { + try { + heartbeat = Instant.parse(hb.toString()); + } catch (RuntimeException ignored) { + // keep default + } + } + return Optional.of( + new ClusterNode( + nodeId.toString(), + String.valueOf(entries.getOrDefault("internalAddress", "")), + heartbeat, + String.valueOf(entries.getOrDefault("role", "BOTH")))); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyJobStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyJobStore.java new file mode 100644 index 000000000..8e9387185 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyJobStore.java @@ -0,0 +1,297 @@ +package stirling.software.proprietary.cluster.valkey; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.ScanOptions; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.cluster.JobStore; +import stirling.software.common.cluster.JobStoreEntry; + +/** + * Valkey-backed {@link JobStore}. Each job is one hash; a reverse index maps fileId to jobId. + * + *

put() atomicity: the hash fields, the per-job TTL, and the reverse-index entries are + * issued inside a single pipelined Redis transaction (MULTI/EXEC). A partial failure cannot leave + * the hash without a TTL or with half the file→job index entries written. + */ +@Component +@RequiredArgsConstructor +@ConditionalOnValkeyBackplane +@Slf4j +public class ValkeyJobStore implements JobStore { + + private static final String JOB_PREFIX = "stirling:job:"; + private static final String FILE_INDEX_PREFIX = "stirling:file2job:"; + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final TypeReference> LIST_STRING = new TypeReference<>() {}; + private static final TypeReference> MAP_STRING = new TypeReference<>() {}; + + private final StringRedisTemplate template; + + @Override + public void put(JobStoreEntry entry, Duration ttl) { + String key = JOB_PREFIX + entry.jobId(); + long ttlMs = ttl.toMillis(); + Map fields = new LinkedHashMap<>(); + fields.put("jobId", entry.jobId()); + fields.put("state", entry.state().name()); + fields.put("owningNodeId", entry.owningNodeId() == null ? "" : entry.owningNodeId()); + if (entry.createdAt() != null) { + fields.put("createdAt", entry.createdAt().toString()); + } + if (entry.completedAt() != null) { + fields.put("completedAt", entry.completedAt().toString()); + } + if (entry.error() != null) { + fields.put("error", entry.error()); + } + fields.put("fileIds", writeJson(entry.fileIds() == null ? List.of() : entry.fileIds())); + fields.put( + "resultMeta", + writeJson(entry.resultMeta() == null ? Map.of() : entry.resultMeta())); + + // Build pipelined MULTI/EXEC so the hash, its TTL, and every reverse-index entry + // commit atomically. + template.execute( + (RedisCallback) + connection -> { + connection.multi(); + byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8); + Map hashBytes = new LinkedHashMap<>(); + for (Map.Entry f : fields.entrySet()) { + hashBytes.put( + f.getKey().getBytes(StandardCharsets.UTF_8), + f.getValue().getBytes(StandardCharsets.UTF_8)); + } + connection.hashCommands().hMSet(keyBytes, hashBytes); + connection.keyCommands().pExpire(keyBytes, ttlMs); + if (entry.fileIds() != null) { + for (String fileId : entry.fileIds()) { + byte[] idxKey = + (FILE_INDEX_PREFIX + fileId) + .getBytes(StandardCharsets.UTF_8); + connection + .stringCommands() + .set( + idxKey, + entry.jobId().getBytes(StandardCharsets.UTF_8)); + connection.keyCommands().pExpire(idxKey, ttlMs); + } + } + connection.exec(); + return null; + }); + } + + @Override + public Optional get(String jobId) { + return readEntry(JOB_PREFIX + jobId); + } + + @Override + public void delete(String jobId) { + // WATCH/MULTI/EXEC: read fileIds INSIDE the watched scope so a concurrent put() that + // adds new fileIds between our read and EXEC aborts the transaction. Without this guard, + // an interleaved put() that grows fileIds would leave orphaned reverse-index entries + // pointing at the deleted jobId until their TTL expires. One retry handles the common + // case; further contention falls through to lazy TTL cleanup (acceptable - this is an + // eviction path, not a correctness primitive). + String jobKey = JOB_PREFIX + jobId; + byte[] jobKeyBytes = jobKey.getBytes(StandardCharsets.UTF_8); + for (int attempt = 0; attempt < 2; attempt++) { + Boolean committed = + template.execute( + (RedisCallback) + connection -> { + connection.watch(jobKeyBytes); + // Read the single fileIds field with hGet rather than + // hGetAll + map.get: hGetAll returns a Map + // whose keys compare by identity, so a fresh + // "fileIds".getBytes() lookup never matches and the reverse + // index would be left orphaned. hGet resolves the field + // server-side. + byte[] fileIdsBytes = + connection + .hashCommands() + .hGet( + jobKeyBytes, + "fileIds" + .getBytes( + StandardCharsets + .UTF_8)); + List keysToDelete = new ArrayList<>(); + keysToDelete.add(jobKeyBytes); + if (fileIdsBytes != null) { + List fileIds = + readJsonList( + new String( + fileIdsBytes, + StandardCharsets.UTF_8), + jobKey); + for (String fileId : fileIds) { + keysToDelete.add( + (FILE_INDEX_PREFIX + fileId) + .getBytes(StandardCharsets.UTF_8)); + } + } + connection.multi(); + for (byte[] key : keysToDelete) { + connection.keyCommands().del(key); + } + List results = connection.exec(); + // exec() returns null when WATCH detected a concurrent + // write; spring-data-redis surfaces this as either null + // or empty depending on the driver path. + return results != null && !results.isEmpty(); + }); + if (Boolean.TRUE.equals(committed)) { + return; + } + } + log.warn( + "JobStore.delete({}) lost two WATCH races to concurrent put(); reverse-index" + + " entries may linger until TTL expiry", + jobId); + } + + @Override + public boolean exists(String jobId) { + Boolean exists = template.hasKey(JOB_PREFIX + jobId); + return Boolean.TRUE.equals(exists); + } + + @Override + public Optional findJobIdByFileId(String fileId) { + return Optional.ofNullable(template.opsForValue().get(FILE_INDEX_PREFIX + fileId)); + } + + @Override + public Collection all() { + // SCAN, not KEYS - KEYS blocks the Valkey server for the duration of the walk. + ScanOptions options = ScanOptions.scanOptions().match(JOB_PREFIX + "*").count(256).build(); + List result = new ArrayList<>(); + try (Cursor cursor = template.scan(options)) { + while (cursor.hasNext()) { + readEntry(cursor.next()).ifPresent(result::add); + } + } + return result; + } + + private Optional readEntry(String key) { + Map entries = template.opsForHash().entries(key); + if (entries == null || entries.isEmpty()) { + return Optional.empty(); + } + Object jobId = entries.get("jobId"); + if (jobId == null) { + return Optional.empty(); + } + Instant createdAt = parseInstant(entries.get("createdAt"), key, "createdAt"); + Instant completedAt = parseInstant(entries.get("completedAt"), key, "completedAt"); + List fileIds = parseList(entries.get("fileIds"), key); + Map resultMeta = parseMap(entries.get("resultMeta"), key); + String stateName = + String.valueOf( + entries.getOrDefault("state", JobStoreEntry.JobState.PENDING.name())); + JobStoreEntry.JobState state; + try { + state = JobStoreEntry.JobState.valueOf(stateName); + } catch (IllegalArgumentException ex) { + log.warn("Unrecognised job state '{}' in {}, defaulting to PENDING", stateName, key); + state = JobStoreEntry.JobState.PENDING; + } + String owningNodeId = String.valueOf(entries.getOrDefault("owningNodeId", "")); + String error = entries.get("error") == null ? null : entries.get("error").toString(); + return Optional.of( + new JobStoreEntry( + jobId.toString(), + state, + owningNodeId, + createdAt, + completedAt, + error, + fileIds, + resultMeta)); + } + + private Instant parseInstant(Object v, String key, String field) { + if (v == null) { + return null; + } + try { + return Instant.parse(v.toString()); + } catch (RuntimeException e) { + log.warn( + "JobStore {} field '{}' has malformed timestamp '{}' - treating as missing", + key, + field, + v); + return null; + } + } + + private List parseList(Object v, String key) { + if (v == null) { + return new ArrayList<>(); + } + return readJsonList(v.toString(), key); + } + + private Map parseMap(Object v, String key) { + if (v == null) { + return new HashMap<>(); + } + try { + return MAPPER.readValue(v.toString(), MAP_STRING); + } catch (JsonProcessingException e) { + log.warn( + "JobStore {} field 'resultMeta' is not valid JSON '{}' - treating as empty", + key, + v); + return new HashMap<>(); + } + } + + private static String writeJson(Object value) { + try { + return MAPPER.writeValueAsString(value); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Failed to JSON-serialize JobStore field", e); + } + } + + private List readJsonList(String json, String key) { + try { + List parsed = MAPPER.readValue(json, LIST_STRING); + return parsed == null ? new ArrayList<>() : parsed; + } catch (JsonProcessingException e) { + log.warn( + "JobStore {} field 'fileIds' is not valid JSON '{}' - treating as empty", + key, + json); + return new ArrayList<>(); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyKeyValueCache.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyKeyValueCache.java new file mode 100644 index 000000000..034189a3c --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyKeyValueCache.java @@ -0,0 +1,61 @@ +package stirling.software.proprietary.cluster.valkey; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.ScanOptions; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; + +import stirling.software.common.cluster.KeyValueCache; + +@Component +@RequiredArgsConstructor +@ConditionalOnValkeyBackplane +public class ValkeyKeyValueCache implements KeyValueCache { + + private static final String PREFIX = "stirling:kv:"; + + private final StringRedisTemplate template; + + @Override + public void put(String namespace, String key, String value, Duration ttl) { + template.opsForValue() + .set(buildKey(namespace, key), value, ttl.toMillis(), TimeUnit.MILLISECONDS); + } + + @Override + public Optional get(String namespace, String key) { + return Optional.ofNullable(template.opsForValue().get(buildKey(namespace, key))); + } + + @Override + public void evict(String namespace, String key) { + template.delete(buildKey(namespace, key)); + } + + @Override + public void evictNamespace(String namespace) { + ScanOptions options = + ScanOptions.scanOptions().match(PREFIX + namespace + ":*").count(256).build(); + List keys = new ArrayList<>(); + try (Cursor cursor = template.scan(options)) { + while (cursor.hasNext()) { + keys.add(cursor.next()); + } + } + if (!keys.isEmpty()) { + template.delete(keys); + } + } + + private String buildKey(String namespace, String key) { + return PREFIX + namespace + ":" + key; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyRateLimitStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyRateLimitStore.java new file mode 100644 index 000000000..0adda83c1 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyRateLimitStore.java @@ -0,0 +1,83 @@ +package stirling.software.proprietary.cluster.valkey; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; + +import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.stereotype.Component; + +import io.github.bucket4j.BucketConfiguration; +import io.github.bucket4j.ConsumptionProbe; +import io.github.bucket4j.distributed.BucketProxy; +import io.github.bucket4j.distributed.ExpirationAfterWriteStrategy; +import io.github.bucket4j.distributed.proxy.ProxyManager; +import io.github.bucket4j.redis.lettuce.Bucket4jLettuce; +import io.lettuce.core.AbstractRedisClient; +import io.lettuce.core.RedisClient; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; + +import stirling.software.common.cluster.RateLimitStore; + +/** + * Valkey-backed token-bucket rate limiting via Bucket4j's Lettuce ProxyManager. The token bucket + * refills continuously and enforces one global limit across nodes, with the same semantics as the + * in-process {@code InProcessRateLimitStore} (which also uses Bucket4j). + */ +@Component +@ConditionalOnValkeyBackplane +public class ValkeyRateLimitStore implements RateLimitStore { + + private static final String PREFIX = "stirling:rl:"; + + private final LettuceConnectionFactory connectionFactory; + private ProxyManager proxyManager; + + public ValkeyRateLimitStore(LettuceConnectionFactory connectionFactory) { + this.connectionFactory = connectionFactory; + } + + @PostConstruct + void initProxyManager() { + AbstractRedisClient client = connectionFactory.getNativeClient(); + if (!(client instanceof RedisClient redisClient)) { + throw new IllegalStateException( + "ValkeyRateLimitStore requires a standalone Lettuce RedisClient; got " + + (client == null ? "null" : client.getClass().getName()) + + " (cluster client not supported by this rate limit impl)"); + } + // Expire idle bucket keys so they do not accumulate forever in Valkey (one key per + // user / API-key / IP). TTL tracks the time to refill the bucket from empty, capped at + // 25h to cover the longest (daily) rate-limit window; an idle bucket evicts after that. + this.proxyManager = + Bucket4jLettuce.casBasedBuilder(redisClient) + .expirationAfterWrite( + ExpirationAfterWriteStrategy.basedOnTimeForRefillingBucketUpToMax( + Duration.ofHours(25))) + .build(); + } + + @PreDestroy + void shutdown() { + proxyManager = null; + } + + @Override + public RateLimitDecision tryConsume(String bucketKey, long capacity, Duration refillPeriod) { + byte[] key = (PREFIX + bucketKey).getBytes(StandardCharsets.UTF_8); + BucketConfiguration cfg = + BucketConfiguration.builder() + .addLimit( + stage -> + stage.capacity(capacity) + .refillGreedy(capacity, refillPeriod)) + .build(); + BucketProxy bucket = proxyManager.builder().build(key, () -> cfg); + ConsumptionProbe probe = bucket.tryConsumeAndReturnRemaining(1); + if (probe.isConsumed()) { + return new RateLimitDecision(true, probe.getRemainingTokens(), 0L); + } + return new RateLimitDecision(false, 0L, probe.getNanosToWaitForRefill()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterLicenseGateTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterLicenseGateTest.java new file mode 100644 index 000000000..49920076d --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterLicenseGateTest.java @@ -0,0 +1,62 @@ +package stirling.software.proprietary.cluster; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import org.junit.jupiter.api.Test; + +class ClusterLicenseGateTest { + + private void injectRunningProOrHigher(ClusterLicenseGate gate, Boolean value) throws Exception { + Field f = ClusterLicenseGate.class.getDeclaredField("runningProOrHigher"); + f.setAccessible(true); + f.set(gate, value); + } + + private void invokeVerify(ClusterLicenseGate gate) throws Throwable { + Method m = ClusterLicenseGate.class.getDeclaredMethod("verifyLicense"); + m.setAccessible(true); + try { + m.invoke(gate); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + + @Test + void serverOrEnterpriseLicense_allowsClusterMode() throws Throwable { + ClusterLicenseGate gate = new ClusterLicenseGate(); + injectRunningProOrHigher(gate, Boolean.TRUE); + assertDoesNotThrow(() -> invokeVerify(gate)); + } + + @Test + void normalLicense_refusesClusterMode_withActionableMessage() throws Exception { + ClusterLicenseGate gate = new ClusterLicenseGate(); + injectRunningProOrHigher(gate, Boolean.FALSE); + IllegalStateException ex = + assertThrows(IllegalStateException.class, () -> invokeVerify(gate)); + String msg = ex.getMessage(); + // The error message must tell the operator exactly what to do. + assertTrue(msg.contains("SERVER"), "message must mention SERVER license tier: " + msg); + assertTrue(msg.contains("ENTERPRISE"), "message must mention ENTERPRISE tier: " + msg); + assertTrue( + msg.contains("stirling.premium.key") || msg.contains("license key"), + "message must explain how to set the license: " + msg); + assertTrue( + msg.contains("cluster.enabled=false"), + "message must offer the opt-out (disable cluster): " + msg); + } + + @Test + void saasFlavor_bypassesGate_whenRunningProOrHigherBeanAbsent() throws Throwable { + // In saas builds the runningProOrHigher bean is absent (@Autowired required=false -> null). + ClusterLicenseGate gate = new ClusterLicenseGate(); + assertDoesNotThrow(() -> invokeVerify(gate)); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterMetricsTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterMetricsTest.java new file mode 100644 index 000000000..d85ee80d1 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterMetricsTest.java @@ -0,0 +1,122 @@ +package stirling.software.proprietary.cluster; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +import stirling.software.common.model.ApplicationProperties; + +/** Verifies every cluster metric is registered and recorder methods write to them. */ +class ClusterMetricsTest { + + private SimpleMeterRegistry registry; + private ClusterMetrics metrics; + private static final String NODE = "test-node"; + + @BeforeEach + void setUp() { + registry = new SimpleMeterRegistry(); + ApplicationProperties props = new ApplicationProperties(); + props.getCluster().getNode().setId(NODE); + metrics = new ClusterMetrics(registry, props); + } + + @Test + void registersAllRequiredMeters() { + assertNotNull(registry.find("stirling_cluster_sticky_miss_total").counter()); + assertNotNull(registry.find("stirling_cluster_ratelimit_rejected_total").counter()); + assertNotNull(registry.find("stirling_cluster_backplane_latency_seconds").timer()); + assertNotNull(registry.find("stirling_cluster_job_wait_seconds").timer()); + Gauge inflight = registry.find("stirling_cluster_jobs_inflight").tag("node", NODE).gauge(); + assertNotNull(inflight, "jobs_inflight gauge with node tag must be registered eagerly"); + } + + @Test + void registersKnownLaneGaugesEagerly() { + for (String lane : new String[] {"FAST", "SLOW", "AI"}) { + Gauge g = registry.find("stirling_cluster_queue_depth").tag("lane", lane).gauge(); + assertNotNull(g, "lane gauge must be eagerly registered for " + lane); + assertEquals(0.0, g.value(), "lane gauge default value must be 0 for " + lane); + } + assertEquals( + 3, + registry.find("stirling_cluster_queue_depth").gauges().size(), + "exactly the three known lane gauges should be registered at boot"); + } + + @Test + void recordStickyMissIncrementsCounter() { + metrics.recordStickyMiss(); + metrics.recordStickyMiss(); + assertEquals(2.0, registry.find("stirling_cluster_sticky_miss_total").counter().count()); + } + + @Test + void recordRateLimitRejectIncrementsCounter() { + metrics.recordRateLimitReject(); + assertEquals( + 1.0, registry.find("stirling_cluster_ratelimit_rejected_total").counter().count()); + } + + @Test + void incrementAndDecrementInflightUpdatesGauge() { + metrics.incrementInflight(); + metrics.incrementInflight(); + metrics.incrementInflight(); + metrics.decrementInflight(); + Gauge gauge = registry.find("stirling_cluster_jobs_inflight").tag("node", NODE).gauge(); + assertEquals(2.0, gauge.value(), "expected 2 inflight after 3 inc / 1 dec"); + } + + @Test + void setQueueDepthUpdatesEagerlyRegisteredLaneGauge() { + metrics.setQueueDepth("FAST", 4); + metrics.setQueueDepth("SLOW", 7); + + Gauge fast = registry.find("stirling_cluster_queue_depth").tag("lane", "FAST").gauge(); + Gauge slow = registry.find("stirling_cluster_queue_depth").tag("lane", "SLOW").gauge(); + assertEquals(4.0, fast.value()); + assertEquals(7.0, slow.value()); + } + + @Test + void setQueueDepthForUnknownLane_lazyRegistersFallbackGauge() { + metrics.setQueueDepth("custom-lane", 5); + Gauge g = registry.find("stirling_cluster_queue_depth").tag("lane", "custom-lane").gauge(); + assertNotNull(g); + assertEquals(5.0, g.value()); + } + + @Test + void setQueueDepthIsIdempotentAcrossCalls() { + metrics.setQueueDepth("FAST", 1); + metrics.setQueueDepth("FAST", 2); + metrics.setQueueDepth("FAST", 9); + + assertEquals( + 1, + registry.find("stirling_cluster_queue_depth").tag("lane", "FAST").gauges().size()); + assertEquals( + 9.0, + registry.find("stirling_cluster_queue_depth").tag("lane", "FAST").gauge().value()); + } + + @Test + void backplaneLatencyTimerAcceptsRecordings() { + metrics.backplaneLatency().record(java.time.Duration.ofMillis(7)); + metrics.backplaneLatency().record(java.time.Duration.ofMillis(11)); + assertEquals( + 2L, registry.find("stirling_cluster_backplane_latency_seconds").timer().count()); + } + + @Test + void jobWaitTimerAcceptsRecordings() { + metrics.jobWaitSeconds().record(java.time.Duration.ofMillis(50)); + assertEquals(1L, registry.find("stirling_cluster_job_wait_seconds").timer().count()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterNodeBootstrapTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterNodeBootstrapTest.java new file mode 100644 index 000000000..7337b9c5d --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterNodeBootstrapTest.java @@ -0,0 +1,115 @@ +package stirling.software.proprietary.cluster; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.time.Duration; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; + +import stirling.software.common.cluster.ClusterNode; +import stirling.software.common.cluster.InstanceRegistry; +import stirling.software.common.model.ApplicationProperties; + +/** Verifies the bootstrap registers / heartbeats / deregisters as expected. */ +class ClusterNodeBootstrapTest { + + private InstanceRegistry registry; + private ApplicationProperties props; + private ClusterNodeBootstrap bootstrap; + + @BeforeEach + void setUp() { + registry = mock(InstanceRegistry.class); + props = new ApplicationProperties(); + props.getCluster().setEnabled(true); + props.getCluster().getNode().setId("node-test-1"); + props.getCluster().getNode().setRole("worker"); + props.getCluster().getNode().setHeartbeatIntervalMs(10_000L); + bootstrap = new ClusterNodeBootstrap(props, registry); + ReflectionTestUtils.setField(bootstrap, "serverPort", 8080); + } + + @Test + void registerOnStartupCallsRegistryWithResolvedNodeId() { + bootstrap.registerOnStartup(); + ArgumentCaptor nodeCaptor = ArgumentCaptor.forClass(ClusterNode.class); + ArgumentCaptor ttlCaptor = ArgumentCaptor.forClass(Duration.class); + verify(registry, times(1)).register(nodeCaptor.capture(), ttlCaptor.capture()); + ClusterNode captured = nodeCaptor.getValue(); + assertEquals("node-test-1", captured.nodeId()); + assertTrue(captured.internalAddress().startsWith("http://")); + assertTrue(captured.internalAddress().endsWith(":8080")); + assertEquals("WORKER", captured.role()); + assertEquals(30L, ttlCaptor.getValue().toSeconds()); + } + + @Test + void registerHonoursExplicitInternalAddress() { + props.getCluster().getNode().setInternalAddress("app-1:8080"); + bootstrap.registerOnStartup(); + ArgumentCaptor nodeCaptor = ArgumentCaptor.forClass(ClusterNode.class); + verify(registry).register(nodeCaptor.capture(), any()); + assertEquals("http://app-1:8080", nodeCaptor.getValue().internalAddress()); + } + + @Test + void registerUsesHttpsSchemeWhenConfigured() { + props.getCluster().getNode().setInternalAddress("app-1:8443"); + props.getCluster().getNode().setScheme("https"); + ClusterNodeBootstrap httpsBootstrap = new ClusterNodeBootstrap(props, registry); + ReflectionTestUtils.setField(httpsBootstrap, "serverPort", 8443); + httpsBootstrap.registerOnStartup(); + ArgumentCaptor nodeCaptor = ArgumentCaptor.forClass(ClusterNode.class); + verify(registry).register(nodeCaptor.capture(), any()); + assertEquals("https://app-1:8443", nodeCaptor.getValue().internalAddress()); + } + + @Test + void heartbeatAfterStartup_callsRegister_forSelfHealing() { + bootstrap.start(); + bootstrap.registerOnStartup(); + bootstrap.heartbeat(); + verify(registry, times(2)) + .register( + any(ClusterNode.class), + org.mockito.ArgumentMatchers.eq(Duration.ofSeconds(30))); + } + + @Test + void smartLifecycleStop_deregisters() { + bootstrap.start(); + bootstrap.registerOnStartup(); + bootstrap.stop(); + verify(registry, times(1)).deregister("node-test-1"); + } + + @Test + void smartLifecycleStop_beforeStartup_isNoop() { + bootstrap.stop(); + verify(registry, never()).deregister(any()); + } + + @Test + void heartbeatAfterStop_doesNotReRegister() { + // Heartbeat-after-stop race: the @Scheduled tick fires during a slow drain. Without + // a guard it would re-register the dead node until TTL expiry. + bootstrap.start(); + bootstrap.registerOnStartup(); + verify(registry, times(1)).register(any(ClusterNode.class), any(Duration.class)); + + bootstrap.stop(); + verify(registry, times(1)).deregister("node-test-1"); + + bootstrap.heartbeat(); + verify(registry, times(1)).register(any(ClusterNode.class), any(Duration.class)); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/MultiNodeClusterScenarioTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/MultiNodeClusterScenarioTest.java new file mode 100644 index 000000000..b78e4ebed --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/MultiNodeClusterScenarioTest.java @@ -0,0 +1,133 @@ +package stirling.software.proprietary.cluster; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import stirling.software.common.cluster.ClusterBackplane; +import stirling.software.common.cluster.JobStore; +import stirling.software.common.cluster.JobStoreEntry; +import stirling.software.common.cluster.KeyValueCache; +import stirling.software.common.cluster.RateLimitStore; +import stirling.software.common.cluster.RateLimitStore.RateLimitDecision; +import stirling.software.common.cluster.inprocess.InProcessJobStore; +import stirling.software.common.cluster.inprocess.InProcessKeyValueCache; +import stirling.software.common.cluster.inprocess.InProcessRateLimitStore; + +/** + * Multi-node contract test using in-process impls shared across two "nodes". Verifies cross-node + * visibility, global rate-limit counters, and cache propagation without requiring Docker. + * Valkey-specific behavior (MULTI/EXEC atomicity, WATCH races, TTL) is in + * LiveValkeyIntegrationTest. + */ +class MultiNodeClusterScenarioTest { + + private JobStore sharedJobStore; + private RateLimitStore sharedRateLimit; + private KeyValueCache sharedCache; + private ClusterBackplane backplaneA; + private ClusterBackplane backplaneB; + + @BeforeEach + void setUp() { + sharedJobStore = new InProcessJobStore(); + sharedRateLimit = new InProcessRateLimitStore(); + sharedCache = new InProcessKeyValueCache(); + backplaneA = constBackplane("node-A", "valkey"); + backplaneB = constBackplane("node-B", "valkey"); + } + + @Test + @DisplayName("async job created on node-A is readable from node-B via shared JobStore") + void jobStatusVisibleCrossNode() { + JobStoreEntry entry = + new JobStoreEntry( + "job-1", + JobStoreEntry.JobState.RUNNING, + "node-A", + Instant.now(), + null, + null, + List.of("file-1"), + Map.of()); + sharedJobStore.put(entry, Duration.ofMinutes(30)); + + Optional seenOnB = sharedJobStore.get("job-1"); + assertTrue(seenOnB.isPresent(), "node-B must see node-A's job in shared JobStore"); + assertEquals("node-A", seenOnB.get().owningNodeId()); + assertEquals(JobStoreEntry.JobState.RUNNING, seenOnB.get().state()); + } + + @Test + @DisplayName("global rate limit - capacity counted once across both nodes") + void rateLimitGlobalAcrossNodes() { + long capacity = 4L; + RateLimitDecision a1 = + sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1)); + RateLimitDecision b1 = + sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1)); + RateLimitDecision a2 = + sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1)); + RateLimitDecision b2 = + sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1)); + RateLimitDecision a3 = + sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1)); + + assertTrue(a1.allowed()); + assertTrue(b1.allowed()); + assertTrue(a2.allowed()); + assertTrue(b2.allowed()); + assertFalse(a3.allowed(), "5th request across both nodes must be rejected (limit=4)"); + } + + @Test + @DisplayName("KeyValueCache populated on A is observed on B; evict on A propagates") + void apiKeyCacheVisibleCrossNode() { + sharedCache.put("apikey", "hash-bob", "bob", Duration.ofSeconds(60)); + assertEquals("bob", sharedCache.get("apikey", "hash-bob").orElse(null)); + sharedCache.evict("apikey", "hash-bob"); + assertFalse(sharedCache.get("apikey", "hash-bob").isPresent()); + } + + @Test + @DisplayName("backplaneType reports 'valkey' on every node; localNodeId is distinct") + void backplaneType() { + assertEquals("valkey", backplaneA.backplaneType()); + assertEquals("valkey", backplaneB.backplaneType()); + assertEquals("node-A", backplaneA.localNodeId()); + assertEquals("node-B", backplaneB.localNodeId()); + assertNotEquals(backplaneA.localNodeId(), backplaneB.localNodeId()); + assertNotNull(backplaneA.localNodeId()); + } + + private ClusterBackplane constBackplane(String nodeId, String type) { + return new ClusterBackplane() { + @Override + public boolean isHealthy() { + return true; + } + + @Override + public String backplaneType() { + return type; + } + + @Override + public String localNodeId() { + return nodeId; + } + }; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveExternalClusterTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveExternalClusterTest.java new file mode 100644 index 000000000..5d4c51ab3 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveExternalClusterTest.java @@ -0,0 +1,237 @@ +package stirling.software.proprietary.cluster.valkey; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.data.redis.core.StringRedisTemplate; + +import stirling.software.common.cluster.ClusterNode; +import stirling.software.common.cluster.DistributedLock; +import stirling.software.common.cluster.JobStoreEntry; +import stirling.software.common.cluster.RateLimitStore.RateLimitDecision; +import stirling.software.common.model.ApplicationProperties; + +/** + * Opt-in live cluster test against an EXTERNAL Valkey/Redis given by {@code + * STIRLING_TEST_VALKEY_URL} (e.g. a managed {@code rediss://} endpoint). Unlike {@link + * LiveValkeyIntegrationTest} (no-auth local container) this drives three independent node stacks + * through the production {@link ValkeyConnectionConfiguration#valkeyConnectionFactory()} bean - so + * a {@code rediss://} URL exercises the real TLS handshake (verifyPeer=FULL) and credential path + * end to end. + * + *

Skips unless the env var is set, so it never runs in normal CI. No secrets are committed. + */ +@EnabledIfEnvironmentVariable(named = "STIRLING_TEST_VALKEY_URL", matches = "rediss?://.+") +class LiveExternalClusterTest { + + private static final int NODES = 3; + private static final String RUN = UUID.randomUUID().toString().substring(0, 8); + + private static final List factories = new ArrayList<>(); + private static final List templates = new ArrayList<>(); + + @BeforeAll + static void connectAllNodes() { + String url = System.getenv("STIRLING_TEST_VALKEY_URL"); + for (int i = 0; i < NODES; i++) { + ApplicationProperties p = new ApplicationProperties(); + p.getCluster().setEnabled(true); + p.getCluster().setBackplane("valkey"); + p.getCluster().getValkey().setUrl(url); + p.getCluster().getNode().setId(nodeId(i)); + // Production bean: parse + credentials + TLS + eager PING handshake (proves reachable). + LettuceConnectionFactory f = + new ValkeyConnectionConfiguration(p).valkeyConnectionFactory(); + factories.add(f); + templates.add(new StringRedisTemplate(f)); + } + } + + @AfterAll + static void disconnect() { + for (LettuceConnectionFactory f : factories) { + try { + f.destroy(); + } catch (RuntimeException ignored) { + // best effort + } + } + } + + private static String nodeId(int i) { + return "ext-" + RUN + "-node-" + (i + 1); + } + + private ApplicationProperties propsForNode(int i) { + ApplicationProperties p = new ApplicationProperties(); + p.getCluster().setEnabled(true); + p.getCluster().setBackplane("valkey"); + p.getCluster().getNode().setId(nodeId(i)); + return p; + } + + @Test + @DisplayName("TLS reachability: every node's backplane reports healthy over the external URL") + void allNodesHealthyOverTls() { + for (int i = 0; i < NODES; i++) { + ValkeyClusterBackplane bp = + new ValkeyClusterBackplane(propsForNode(i), templates.get(i)); + assertTrue(bp.isHealthy(), nodeId(i) + " must reach the external Valkey (PING)"); + assertEquals(nodeId(i), bp.localNodeId()); + } + } + + @Test + @DisplayName("3-node registry: each node sees all peers; deregister drops one cluster-wide") + void threeNodeRegistryConverges() { + List regs = new ArrayList<>(); + for (int i = 0; i < NODES; i++) { + regs.add(new ValkeyInstanceRegistry(templates.get(i))); + } + for (int i = 0; i < NODES; i++) { + regs.get(i) + .register( + new ClusterNode( + nodeId(i), "10.0.0." + i + ":8080", Instant.now(), "BOTH"), + Duration.ofSeconds(30)); + } + try { + // Node 0's view must include all three registrations made on three connections. + for (int i = 0; i < NODES; i++) { + final String id = nodeId(i); + boolean seenByNode0 = + regs.get(0).activeNodes().stream().anyMatch(n -> id.equals(n.nodeId())); + assertTrue(seenByNode0, "node-0 must see " + id + " registered by another node"); + } + regs.get(1).deregister(nodeId(2)); + assertFalse( + regs.get(0).lookup(nodeId(2)).isPresent(), + "deregister on node-1 must be visible from node-0"); + } finally { + regs.get(0).deregister(nodeId(0)); + regs.get(1).deregister(nodeId(1)); + } + } + + @Test + @DisplayName( + "Cross-node JobStore: put on node-0 visible on node-1/2; delete on node-2 clears it") + void jobStoreVisibleAcrossNodes() { + ValkeyJobStore s0 = new ValkeyJobStore(templates.get(0)); + ValkeyJobStore s1 = new ValkeyJobStore(templates.get(1)); + ValkeyJobStore s2 = new ValkeyJobStore(templates.get(2)); + String jobId = "ext-job-" + RUN; + String fileId = "ext-file-" + RUN; + + s0.put( + new JobStoreEntry( + jobId, + JobStoreEntry.JobState.RUNNING, + nodeId(0), + Instant.now(), + null, + null, + List.of(fileId), + Map.of("k", "v")), + Duration.ofSeconds(60)); + + Optional onNode1 = s1.get(jobId); + assertTrue(onNode1.isPresent(), "node-1 must see node-0's job write"); + assertEquals(nodeId(0), onNode1.get().owningNodeId()); + assertEquals(jobId, s2.findJobIdByFileId(fileId).orElse(null), "reverse index visible too"); + + s2.delete(jobId); + assertFalse(s0.exists(jobId), "delete on node-2 must clear the hash for node-0"); + assertFalse( + s1.findJobIdByFileId(fileId).isPresent(), + "delete on node-2 must clear the reverse index for node-1"); + } + + @Test + @DisplayName("Global rate limit: ONE shared budget enforced across all three nodes") + void rateLimitGlobalAcrossThreeNodes() { + List stores = new ArrayList<>(); + for (int i = 0; i < NODES; i++) { + ValkeyRateLimitStore s = new ValkeyRateLimitStore(factories.get(i)); + s.initProxyManager(); + stores.add(s); + } + String key = "ext-rl-" + RUN; + long capacity = 6; + AtomicInteger allowed = new AtomicInteger(); + for (int i = 0; i < 12; i++) { + RateLimitDecision d = + stores.get(i % NODES).tryConsume(key, capacity, Duration.ofSeconds(60)); + if (d.allowed()) { + allowed.incrementAndGet(); + } + } + assertEquals( + capacity, + allowed.get(), + "exactly the global capacity must be allowed across all three nodes"); + } + + @Test + @DisplayName("Distributed lock: held by node-0 excludes node-1/2; stale release cannot steal") + void distributedLockAcrossNodes() throws InterruptedException { + ValkeyDistributedLock l0 = new ValkeyDistributedLock(templates.get(0)); + ValkeyDistributedLock l1 = new ValkeyDistributedLock(templates.get(1)); + ValkeyDistributedLock l2 = new ValkeyDistributedLock(templates.get(2)); + String key = "ext-lock-" + RUN; + + Optional a = l0.tryAcquire(key, Duration.ofSeconds(30)); + assertTrue(a.isPresent()); + assertFalse(l1.tryAcquire(key, Duration.ofSeconds(30)).isPresent(), "node-1 excluded"); + assertFalse(l2.tryAcquire(key, Duration.ofSeconds(30)).isPresent(), "node-2 excluded"); + a.get().release(); + Optional b = l1.tryAcquire(key, Duration.ofSeconds(30)); + assertTrue(b.isPresent(), "node-1 acquires after node-0 releases"); + + // Short lease that expires, then node-2 takes it; node-1's stale release must not steal. + String key2 = "ext-lock2-" + RUN; + Optional shortHeld = + l1.tryAcquire(key2, Duration.ofMillis(500)); + assertTrue(shortHeld.isPresent()); + Thread.sleep(900); + Optional stolen = l2.tryAcquire(key2, Duration.ofSeconds(30)); + assertTrue(stolen.isPresent(), "node-2 acquires after node-1's lease expires"); + shortHeld.get().release(); // value-checked no-op + assertFalse( + l0.tryAcquire(key2, Duration.ofMillis(200)).isPresent(), + "node-2 still holds; node-1's stale release must not have stolen it"); + + b.get().release(); + stolen.get().release(); + } + + @Test + @DisplayName("KeyValueCache: put on node-0 readable on node-1; evict visible on node-2") + void keyValueCacheAcrossNodes() { + ValkeyKeyValueCache c0 = new ValkeyKeyValueCache(templates.get(0)); + ValkeyKeyValueCache c1 = new ValkeyKeyValueCache(templates.get(1)); + ValkeyKeyValueCache c2 = new ValkeyKeyValueCache(templates.get(2)); + String field = "ext-cache-" + RUN; + + c0.put("apikey", field, "alice", Duration.ofSeconds(60)); + assertEquals("alice", c1.get("apikey", field).orElse(null), "node-1 reads node-0's cache"); + c2.evict("apikey", field); + assertFalse(c0.get("apikey", field).isPresent(), "evict on node-2 visible on node-0"); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyAuthIntegrationTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyAuthIntegrationTest.java new file mode 100644 index 000000000..133ef35f6 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyAuthIntegrationTest.java @@ -0,0 +1,143 @@ +package stirling.software.proprietary.cluster.valkey; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import stirling.software.common.model.ApplicationProperties; + +/** + * Live AUTH coverage against a password-protected Valkey. The main {@link + * LiveValkeyIntegrationTest} runs against a no-auth instance, so the credential-bearing URL path + * (parse -> RedisStandaloneConfiguration -> real AUTH handshake) was otherwise unexercised. Drives + * the full production bean method {@code valkeyConnectionFactory()} so the parse, credential + * wiring, and eager-handshake all run exactly as at boot. + */ +@Testcontainers +@EnabledIf("isDockerAvailable") +class LiveValkeyAuthIntegrationTest { + + private static final String PASSWORD = "s3cr3tpw"; + + // @Container is intentionally NOT used: lifecycle is managed manually so the container can be + // shared across the password-only and ACL-user cases without a per-method restart. + static final GenericContainer VALKEY = + new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine")) + .withExposedPorts(6379) + .withCommand("valkey-server", "--requirepass", PASSWORD); + + static boolean isDockerAvailable() { + return DockerClientFactory.instance().isDockerAvailable(); + } + + @org.junit.jupiter.api.BeforeAll + static void start() { + VALKEY.start(); + } + + @org.junit.jupiter.api.AfterAll + static void stop() { + VALKEY.stop(); + } + + private static String host() { + return VALKEY.getHost(); + } + + private static int port() { + return VALKEY.getMappedPort(6379); + } + + private ApplicationProperties propsWithUrl(String url) { + ApplicationProperties p = new ApplicationProperties(); + p.getCluster().setEnabled(true); + p.getCluster().setBackplane("valkey"); + p.getCluster().getValkey().setUrl(url); + p.getCluster().getNode().setId("auth-test"); + return p; + } + + @Test + @DisplayName("password-only URL (redis://:pw@host) authenticates and round-trips a key") + void passwordOnlyAuthWorks() { + String url = "redis://:" + PASSWORD + "@" + host() + ":" + port(); + LettuceConnectionFactory factory = + new ValkeyConnectionConfiguration(propsWithUrl(url)).valkeyConnectionFactory(); + try { + StringRedisTemplate t = new StringRedisTemplate(factory); + t.opsForValue().set("auth:pwonly", "v1"); + assertEquals( + "v1", + t.opsForValue().get("auth:pwonly"), + "password-only AUTH must succeed and the key must round-trip"); + } finally { + factory.destroy(); + } + } + + @Test + @DisplayName("user:password URL (ACL named user) authenticates and round-trips a key") + void namedUserAuthWorks() throws Exception { + // Default user is password-protected; create a named ACL user to exercise two-arg AUTH. + Container.ExecResult res = + VALKEY.execInContainer( + "valkey-cli", + "-a", + PASSWORD, + "ACL", + "SETUSER", + "alice", + "on", + ">alicepass", + "~*", + "+@all"); + assertEquals(0, res.getExitCode(), "ACL SETUSER failed: " + res.getStderr()); + + String url = "redis://alice:alicepass@" + host() + ":" + port(); + LettuceConnectionFactory factory = + new ValkeyConnectionConfiguration(propsWithUrl(url)).valkeyConnectionFactory(); + try { + StringRedisTemplate t = new StringRedisTemplate(factory); + t.opsForValue().set("auth:named", "v2"); + assertEquals( + "v2", + t.opsForValue().get("auth:named"), + "named-user AUTH must succeed and the key must round-trip"); + } finally { + factory.destroy(); + } + } + + @Test + @DisplayName( + "wrong password → fast IllegalStateException (real WRONGPASS short-circuits retries)") + void wrongPasswordFailsFast() { + String url = "redis://:wrong-" + PASSWORD + "@" + host() + ":" + port(); + ValkeyConnectionConfiguration cfg = new ValkeyConnectionConfiguration(propsWithUrl(url)); + + long start = System.nanoTime(); + IllegalStateException ex = + assertThrows(IllegalStateException.class, cfg::valkeyConnectionFactory); + long elapsedMs = (System.nanoTime() - start) / 1_000_000; + + assertTrue( + ex.getMessage().toLowerCase().contains("authentication failed"), + "real Valkey WRONGPASS must be classified as an auth failure; got: " + + ex.getMessage()); + // The retry loop is 10 x 3s; a recognised auth failure must abort well before that. + assertTrue( + elapsedMs < 10_000, + "auth failure must short-circuit the 30s retry loop; elapsed=" + elapsedMs + " ms"); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyChaosTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyChaosTest.java new file mode 100644 index 000000000..f619272c1 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyChaosTest.java @@ -0,0 +1,118 @@ +package stirling.software.proprietary.cluster.valkey; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.springframework.dao.DataAccessException; +import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import stirling.software.common.model.ApplicationProperties; + +/** + * Live failure-injection: a frozen (network-black-holed) Valkey must NOT stall hot-path commands + * for Lettuce's 60s default. {@link ValkeyConnectionConfiguration} pins a 2s command timeout, so a + * paused server must surface an error in seconds, and the connection must recover when it returns. + * + *

Uses {@code docker pause}/{@code unpause} (TCP stays ESTABLISHED but the server never replies) + * to reproduce a partition rather than {@code stop} (which would fail fast with + * connection-refused). + */ +@Testcontainers +@EnabledIf("isDockerAvailable") +class LiveValkeyChaosTest { + + @Container + static final GenericContainer VALKEY = + new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine")) + .withExposedPorts(6379); + + static boolean isDockerAvailable() { + return DockerClientFactory.instance().isDockerAvailable(); + } + + private boolean paused; + + private LettuceConnectionFactory productionFactory() { + ApplicationProperties p = new ApplicationProperties(); + p.getCluster().setEnabled(true); + p.getCluster().setBackplane("valkey"); + p.getCluster() + .getValkey() + .setUrl("redis://" + VALKEY.getHost() + ":" + VALKEY.getMappedPort(6379)); + p.getCluster().getNode().setId("chaos"); + // Built via the production bean so the real 2s commandTimeout is in effect. + return new ValkeyConnectionConfiguration(p).valkeyConnectionFactory(); + } + + private void pause() { + DockerClientFactory.lazyClient().pauseContainerCmd(VALKEY.getContainerId()).exec(); + paused = true; + } + + private void unpause() { + DockerClientFactory.lazyClient().unpauseContainerCmd(VALKEY.getContainerId()).exec(); + paused = false; + } + + @AfterEach + void ensureUnpaused() { + if (paused) { + try { + unpause(); + } catch (RuntimeException ignored) { + // container teardown will handle it + } + } + } + + @Test + @DisplayName("frozen Valkey fails a command in seconds (2s timeout), not Lettuce's 60s default") + void commandTimeoutFiresUnderPartition() { + LettuceConnectionFactory factory = productionFactory(); + try { + StringRedisTemplate t = new StringRedisTemplate(factory); + t.opsForValue().set("chaos:k", "before"); + assertEquals("before", t.opsForValue().get("chaos:k")); + + pause(); + long start = System.nanoTime(); + // DataAccessException is spring-data-redis's wrapper for the Lettuce timeout. + assertThrows(DataAccessException.class, () -> t.opsForValue().get("chaos:k")); + long elapsedMs = (System.nanoTime() - start) / 1_000_000; + assertTrue( + elapsedMs < 15_000, + "command must abort on the ~2s timeout, not hang on the 60s default; elapsed=" + + elapsedMs + + " ms"); + + unpause(); + // After the partition heals the client must recover (Lettuce reconnects lazily). + String recovered = null; + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline) { + try { + recovered = t.opsForValue().get("chaos:k"); + break; + } catch (RuntimeException retry) { + Thread.sleep(250); + } + } + assertEquals("before", recovered, "connection must recover after the partition heals"); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } finally { + factory.destroy(); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyIntegrationTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyIntegrationTest.java new file mode 100644 index 000000000..c26528a71 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyIntegrationTest.java @@ -0,0 +1,519 @@ +package stirling.software.proprietary.cluster.valkey; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.springframework.data.redis.connection.RedisStandaloneConfiguration; +import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import stirling.software.common.cluster.ClusterNode; +import stirling.software.common.cluster.DistributedLock; +import stirling.software.common.cluster.JobStoreEntry; +import stirling.software.common.cluster.RateLimitStore.RateLimitDecision; +import stirling.software.common.model.ApplicationProperties; + +/** + * Live integration tests against a real Valkey instance, started by Testcontainers. The + * {@code @EnabledIf} guard probes the Docker daemon via {@link + * DockerClientFactory#isDockerAvailable()} (non-throwing) so the suite skips cleanly when Docker is + * unavailable - without that guard, {@code @Testcontainers} would throw {@code initializationError} + * (test FAILURE, not skip) on CI runners without Docker. + */ +@Testcontainers +@EnabledIf("isDockerAvailable") +class LiveValkeyIntegrationTest { + + @Container + static final GenericContainer VALKEY = + new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine")) + .withExposedPorts(6379); + + static boolean isDockerAvailable() { + return DockerClientFactory.instance().isDockerAvailable(); + } + + private static LettuceConnectionFactory factoryA; + private static LettuceConnectionFactory factoryB; + private static StringRedisTemplate templateA; + private static StringRedisTemplate templateB; + + @BeforeAll + static void connect() { + String host = VALKEY.getHost(); + int port = VALKEY.getMappedPort(6379); + factoryA = new LettuceConnectionFactory(new RedisStandaloneConfiguration(host, port)); + factoryA.afterPropertiesSet(); + factoryB = new LettuceConnectionFactory(new RedisStandaloneConfiguration(host, port)); + factoryB.afterPropertiesSet(); + templateA = new StringRedisTemplate(factoryA); + templateB = new StringRedisTemplate(factoryB); + templateA.getConnectionFactory().getConnection().serverCommands().flushAll(); + } + + @AfterAll + static void disconnect() { + if (factoryA != null) factoryA.destroy(); + if (factoryB != null) factoryB.destroy(); + } + + @Test + @DisplayName("Valkey reachable and isHealthy() = true after PING round-trip") + void backplaneHealthy() { + ApplicationProperties propsA = newProps("node-A"); + ValkeyClusterBackplane bp = new ValkeyClusterBackplane(propsA, templateA); + assertEquals("valkey", bp.backplaneType()); + assertEquals("node-A", bp.localNodeId()); + assertTrue(bp.isHealthy(), "Valkey must be reachable in the Testcontainers instance"); + } + + @Test + @DisplayName("JobStore put on connection A, get on connection B reads the same entry") + void jobStoreCrossConnectionVisibility() { + ValkeyJobStore storeA = new ValkeyJobStore(templateA); + ValkeyJobStore storeB = new ValkeyJobStore(templateB); + + JobStoreEntry entry = + new JobStoreEntry( + "live-job-1", + JobStoreEntry.JobState.RUNNING, + "node-A", + Instant.now(), + null, + null, + List.of("live-file-1"), + Map.of("k", "v")); + storeA.put(entry, Duration.ofSeconds(30)); + + Optional seen = storeB.get("live-job-1"); + assertTrue(seen.isPresent(), "storeB on different connection must see storeA's write"); + assertEquals("node-A", seen.get().owningNodeId()); + assertEquals(JobStoreEntry.JobState.RUNNING, seen.get().state()); + assertEquals("live-job-1", storeB.findJobIdByFileId("live-file-1").orElse(null)); + } + + @Test + @DisplayName("JobStore entry expires after the configured duration") + void jobStoreTtlExpires() throws InterruptedException { + ValkeyJobStore store = new ValkeyJobStore(templateA); + store.put( + new JobStoreEntry( + "ttl-job", + JobStoreEntry.JobState.PENDING, + "node-A", + Instant.now(), + null, + null, + List.of(), + Map.of()), + Duration.ofSeconds(2)); + assertTrue(store.exists("ttl-job")); + long deadline = System.currentTimeMillis() + 3000; + boolean expired = false; + while (System.currentTimeMillis() < deadline) { + if (!store.exists("ttl-job")) { + expired = true; + break; + } + Thread.sleep(100); + } + assertTrue(expired, "entry should TTL-expire within 3 s of a 2 s TTL"); + } + + @Test + @DisplayName("KeyValueCache propagates across connections; evict observed cross-connection") + void keyValueCacheCrossConnection() { + ValkeyKeyValueCache cacheA = new ValkeyKeyValueCache(templateA); + ValkeyKeyValueCache cacheB = new ValkeyKeyValueCache(templateB); + + cacheA.put("apikey", "hash-bob", "bob", Duration.ofSeconds(30)); + assertEquals("bob", cacheB.get("apikey", "hash-bob").orElse(null)); + + cacheA.evict("apikey", "hash-bob"); + assertFalse(cacheB.get("apikey", "hash-bob").isPresent()); + } + + @Test + @DisplayName("RateLimitStore enforces ONE global budget across two instances") + void rateLimitGlobalAcrossInstances() { + ValkeyRateLimitStore storeA = newRateLimitStore(factoryA); + ValkeyRateLimitStore storeB = newRateLimitStore(factoryB); + String key = "live-user:alice"; + long capacity = 4; + + AtomicInteger allowed = new AtomicInteger(); + for (int i = 0; i < 8; i++) { + // alternate consumers + var store = (i % 2 == 0) ? storeA : storeB; + RateLimitDecision d = store.tryConsume(key, capacity, Duration.ofSeconds(30)); + if (d.allowed()) allowed.incrementAndGet(); + } + assertEquals( + 4, + allowed.get(), + "exactly 4 (the global capacity) must be allowed across both instances"); + } + + @Test + @DisplayName("DistributedLock excludes a second acquirer on a different connection") + void distributedLockMutualExclusion() { + ValkeyDistributedLock lockA = new ValkeyDistributedLock(templateA); + ValkeyDistributedLock lockB = new ValkeyDistributedLock(templateB); + + Optional heldByA = + lockA.tryAcquire("election-X", Duration.ofSeconds(30)); + assertTrue(heldByA.isPresent()); + + Optional heldByB = + lockB.tryAcquire("election-X", Duration.ofSeconds(30)); + assertFalse(heldByB.isPresent(), "second acquirer must fail while A holds the lock"); + + heldByA.get().release(); + + // After release, B can acquire + Optional retry = + lockB.tryAcquire("election-X", Duration.ofSeconds(30)); + assertTrue(retry.isPresent()); + retry.get().release(); + } + + @Test + @DisplayName("renew() extends the lease TTL well beyond the original") + void distributedLockRenewExtendsLease() { + ValkeyDistributedLock lock = new ValkeyDistributedLock(templateA); + String key = "renew-" + java.util.UUID.randomUUID(); + Optional held = lock.tryAcquire(key, Duration.ofSeconds(1)); + assertTrue(held.isPresent()); + + assertTrue(held.get().renew(Duration.ofSeconds(30)), "renew on a held lock must succeed"); + Long ttlMs = + templateA.getExpire( + "stirling:lock:" + key, java.util.concurrent.TimeUnit.MILLISECONDS); + assertNotNull(ttlMs); + assertTrue( + ttlMs > 1500 && ttlMs <= 30_000, + "renew must reset TTL to the new 30s lease, got " + ttlMs + " ms"); + held.get().release(); + } + + @Test + @DisplayName("value-check prevents a stale owner from releasing/renewing a re-acquired lock") + void distributedLockStealPrevention() throws InterruptedException { + ValkeyDistributedLock lockA = new ValkeyDistributedLock(templateA); + ValkeyDistributedLock lockB = new ValkeyDistributedLock(templateB); + String key = "steal-" + java.util.UUID.randomUUID(); + + Optional a = lockA.tryAcquire(key, Duration.ofMillis(500)); + assertTrue(a.isPresent()); + + // Let A's 500ms lease TTL-expire so Valkey drops the key, then B takes a fresh lock. + Thread.sleep(800); + Optional b = lockB.tryAcquire(key, Duration.ofSeconds(30)); + assertTrue(b.isPresent(), "B must acquire after A's lease expired"); + + // A's stale handle (different UUID value) must touch neither B's renew nor B's key. + assertFalse( + a.get().renew(Duration.ofSeconds(30)), + "stale owner must not be able to renew a lock now owned by B"); + a.get().release(); // value-checked DEL: must be a no-op, must NOT delete B's key + + assertFalse( + lockA.tryAcquire(key, Duration.ofMillis(100)).isPresent(), + "B must still hold the lock; A's stale release must not have stolen it"); + b.get().release(); + } + + @Test + @DisplayName("register is atomic (hash + TTL committed together, no orphan keys on crash)") + void registryRegisterIsAtomic() { + ValkeyInstanceRegistry reg = new ValkeyInstanceRegistry(templateA); + ClusterNode node = + new ClusterNode( + "atomic-node-" + java.util.UUID.randomUUID(), + "10.0.0.99:8080", + Instant.now(), + "BOTH"); + reg.register(node, Duration.ofSeconds(30)); + + // TTL must be positive; -1 would mean EXPIRE did not commit inside MULTI/EXEC. + Long ttlMs = + templateA.getExpire( + "stirling:nodes:" + node.nodeId(), + java.util.concurrent.TimeUnit.MILLISECONDS); + assertNotNull(ttlMs); + assertTrue( + ttlMs > 0 && ttlMs <= 30_000, + "register() must atomically arm TTL; expected (0, 30000] ms, got " + ttlMs); + + Optional seen = reg.lookup(node.nodeId()); + assertTrue(seen.isPresent(), "hash fields must be visible after atomic register()"); + assertEquals("10.0.0.99:8080", seen.get().internalAddress()); + + reg.deregister(node.nodeId()); + } + + @Test + @DisplayName("register on connection A is visible from connection B") + void registryCrossConnection() { + ValkeyInstanceRegistry regA = new ValkeyInstanceRegistry(templateA); + ValkeyInstanceRegistry regB = new ValkeyInstanceRegistry(templateB); + + ClusterNode node = new ClusterNode("live-node-7", "10.0.0.7:8080", Instant.now(), "BOTH"); + regA.register(node, Duration.ofSeconds(30)); + + Optional seen = regB.lookup("live-node-7"); + assertTrue(seen.isPresent()); + assertEquals("10.0.0.7:8080", seen.get().internalAddress()); + + boolean inActive = + regB.activeNodes().stream().anyMatch(n -> "live-node-7".equals(n.nodeId())); + assertTrue(inActive); + + regA.deregister("live-node-7"); + assertFalse(regB.lookup("live-node-7").isPresent()); + } + + @Test + @DisplayName("Bucket4j: no fixed-window boundary doubling (parity with in-process semantics)") + void rateLimitNoBoundaryDoubling() throws InterruptedException { + ValkeyRateLimitStore store = newRateLimitStore(factoryA); + String key = "boundary-" + java.util.UUID.randomUUID(); + long capacity = 5; + Duration window = Duration.ofMillis(500); + + int firstAllowed = 0; + for (int i = 0; i < 10; i++) { + if (store.tryConsume(key, capacity, window).allowed()) firstAllowed++; + } + assertEquals(capacity, firstAllowed, "must allow exactly capacity tokens initially"); + + Thread.sleep(window.toMillis() + 50); + int secondAllowed = 0; + long start = System.nanoTime(); + for (int i = 0; i < 20 && (System.nanoTime() - start) < 20_000_000L; i++) { + if (store.tryConsume(key, capacity, window).allowed()) secondAllowed++; + } + assertTrue( + secondAllowed <= capacity, + "token-bucket must not let a fresh full capacity be consumed instantly across" + + " the boundary; got " + + secondAllowed); + } + + private ValkeyRateLimitStore newRateLimitStore(LettuceConnectionFactory factory) { + ValkeyRateLimitStore store = new ValkeyRateLimitStore(factory); + store.initProxyManager(); + return store; + } + + @Test + @DisplayName("JobStore put is atomic (hash + TTL + reverse index visible together)") + void jobStorePutIsAtomic() { + ValkeyJobStore store = new ValkeyJobStore(templateA); + String jobId = "atomic-job-" + java.util.UUID.randomUUID(); + String fileId = "atomic-file-" + java.util.UUID.randomUUID(); + store.put( + new JobStoreEntry( + jobId, + JobStoreEntry.JobState.PENDING, + "node-A", + Instant.now(), + null, + null, + List.of(fileId), + Map.of("k", "v")), + Duration.ofSeconds(30)); + + assertTrue(store.exists(jobId), "hash must be visible after put"); + Long jobTtl = + templateA.getExpire( + "stirling:job:" + jobId, java.util.concurrent.TimeUnit.MILLISECONDS); + assertNotNull(jobTtl); + assertTrue(jobTtl > 0, "hash must have TTL armed inside the same transaction"); + assertEquals(jobId, store.findJobIdByFileId(fileId).orElse(null)); + Long indexTtl = + templateA.getExpire( + "stirling:file2job:" + fileId, java.util.concurrent.TimeUnit.MILLISECONDS); + assertNotNull(indexTtl); + assertTrue(indexTtl > 0, "reverse index must also have TTL armed"); + } + + @Test + @DisplayName( + "JobStore.delete(): WATCH aborts when put() races between read and EXEC, no orphaned" + + " reverse-index entries") + void jobStoreDeleteWatchRaceRetriesAndCleansUp() { + ValkeyJobStore store = new ValkeyJobStore(templateA); + String jobId = "watch-race-job-" + java.util.UUID.randomUUID(); + String originalFile = "orig-file-" + java.util.UUID.randomUUID(); + String newFile = "new-file-" + java.util.UUID.randomUUID(); + + store.put( + new JobStoreEntry( + jobId, + JobStoreEntry.JobState.RUNNING, + "node-A", + Instant.now(), + null, + null, + List.of(originalFile), + Map.of()), + Duration.ofSeconds(30)); + + // Simulate the race: between delete()'s WATCH read and EXEC, add a new fileId. + // The first EXEC aborts; the retry catches the new fileId and deletes both entries. + Thread mutator = + new Thread( + () -> { + try { + Thread.sleep(20); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + store.put( + new JobStoreEntry( + jobId, + JobStoreEntry.JobState.RUNNING, + "node-A", + Instant.now(), + null, + null, + List.of(originalFile, newFile), + Map.of()), + Duration.ofSeconds(30)); + }); + mutator.start(); + + store.delete(jobId); + try { + mutator.join(2000); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + + boolean hashGone = !store.exists(jobId); + boolean origIndexGone = !store.findJobIdByFileId(originalFile).isPresent(); + boolean newIndexGone = !store.findJobIdByFileId(newFile).isPresent(); + if (hashGone) { + assertTrue( + origIndexGone, + "if hash is deleted, original reverse-index entry must also be gone"); + assertTrue( + newIndexGone, + "if hash is deleted after the racing put(), the WATCH retry must catch the" + + " new fileId and delete its reverse-index entry too"); + } else { + assertEquals(jobId, store.findJobIdByFileId(originalFile).orElse(null)); + assertEquals(jobId, store.findJobIdByFileId(newFile).orElse(null)); + } + } + + @Test + @DisplayName("JobStore.delete() removes hash AND every reverse-index entry atomically") + void jobStoreDeleteRemovesReverseIndexEntries() { + ValkeyJobStore store = new ValkeyJobStore(templateA); + String jobId = "del-atomic-job-" + java.util.UUID.randomUUID(); + String fileA = "del-atomic-fileA-" + java.util.UUID.randomUUID(); + String fileB = "del-atomic-fileB-" + java.util.UUID.randomUUID(); + store.put( + new JobStoreEntry( + jobId, + JobStoreEntry.JobState.COMPLETE, + "node-A", + Instant.now(), + Instant.now(), + null, + List.of(fileA, fileB), + Map.of()), + Duration.ofSeconds(30)); + assertTrue(store.exists(jobId)); + assertEquals(jobId, store.findJobIdByFileId(fileA).orElse(null)); + assertEquals(jobId, store.findJobIdByFileId(fileB).orElse(null)); + + store.delete(jobId); + + // Both the main hash AND every reverse-index entry must be gone; dangling reverse-index + // entries would cause findJobIdByFileId() to return a deleted jobId. + assertFalse(store.exists(jobId), "main hash must be deleted"); + assertFalse( + store.findJobIdByFileId(fileA).isPresent(), + "reverse-index entry for fileA must not survive delete()"); + assertFalse( + store.findJobIdByFileId(fileB).isPresent(), + "reverse-index entry for fileB must not survive delete()"); + assertFalse( + Boolean.TRUE.equals(templateA.hasKey("stirling:file2job:" + fileA)), + "raw reverse-index key for fileA must not survive delete()"); + assertFalse( + Boolean.TRUE.equals(templateA.hasKey("stirling:file2job:" + fileB)), + "raw reverse-index key for fileB must not survive delete()"); + } + + @Test + @DisplayName("JobStore.all() walks the keyspace via SCAN, not KEYS") + void jobStoreAllUsesScanNonBlocking() { + ValkeyJobStore store = new ValkeyJobStore(templateA); + for (int i = 0; i < 15; i++) { + store.put( + new JobStoreEntry( + "scan-job-" + i, + JobStoreEntry.JobState.PENDING, + "node-A", + Instant.now(), + null, + null, + List.of(), + Map.of()), + Duration.ofSeconds(30)); + } + long observed = store.all().stream().filter(e -> e.jobId().startsWith("scan-job-")).count(); + assertTrue( + observed >= 15, + "SCAN-based all() must surface every inserted job, saw " + observed); + } + + @Test + @DisplayName("Valkey unreachable yields isHealthy() = false") + void unreachableBackplaneReportsUnhealthy() { + RedisStandaloneConfiguration cfg = new RedisStandaloneConfiguration("localhost", 16400); + LettuceConnectionFactory dead = new LettuceConnectionFactory(cfg); + dead.afterPropertiesSet(); + try { + StringRedisTemplate t = new StringRedisTemplate(dead); + ValkeyClusterBackplane bp = new ValkeyClusterBackplane(newProps("orphan"), t); + assertFalse(bp.isHealthy(), "isHealthy must be false when Valkey is unreachable"); + } finally { + dead.destroy(); + } + } + + private ApplicationProperties newProps(String nodeId) { + ApplicationProperties p = new ApplicationProperties(); + p.getCluster().setEnabled(true); + p.getCluster().setBackplane("valkey"); + p.getCluster() + .getValkey() + .setUrl("redis://" + VALKEY.getHost() + ":" + VALKEY.getMappedPort(6379)); + p.getCluster().getNode().setId(nodeId); + return p; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplaneTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplaneTest.java new file mode 100644 index 000000000..e406e5fa4 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplaneTest.java @@ -0,0 +1,61 @@ +package stirling.software.proprietary.cluster.valkey; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.StringRedisTemplate; + +import stirling.software.common.model.ApplicationProperties; + +/** + * S3 regression: {@link ValkeyClusterBackplane#isHealthy()} must route through {@code + * template.execute(...)} so the borrowed connection is always returned to the pool. Calling {@code + * getConnectionFactory().getConnection()} directly would leak the connection on every k8s liveness + * probe tick and exhaust the pool under monitoring load. + */ +class ValkeyClusterBackplaneTest { + + @Test + void isHealthy_routesThroughTemplateExecute_andDoesNotTouchConnectionFactoryDirectly() { + StringRedisTemplate template = mock(StringRedisTemplate.class); + when(template.execute(any(RedisCallback.class))).thenReturn("PONG"); + + ApplicationProperties props = new ApplicationProperties(); + props.getCluster().getNode().setId("n-1"); + ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template); + + assertTrue(bp.isHealthy()); + verify(template, times(1)).execute(any(RedisCallback.class)); + // Critical: never bypass the template's connection management. + verify(template, never()).getConnectionFactory(); + } + + @Test + void isHealthy_returnsFalseWhenExecuteThrows() { + StringRedisTemplate template = mock(StringRedisTemplate.class); + when(template.execute(any(RedisCallback.class))).thenThrow(new RuntimeException("boom")); + + ApplicationProperties props = new ApplicationProperties(); + props.getCluster().getNode().setId("n-1"); + ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template); + + assertFalse(bp.isHealthy()); + } + + @Test + void shouldRunLocalCleanup_returnsFalse_valkeyOwnsTtlEviction() { + StringRedisTemplate template = mock(StringRedisTemplate.class); + ApplicationProperties props = new ApplicationProperties(); + props.getCluster().getNode().setId("n-1"); + ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template); + assertFalse(bp.shouldRunLocalCleanup()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfigurationTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfigurationTest.java new file mode 100644 index 000000000..5602f01f6 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfigurationTest.java @@ -0,0 +1,298 @@ +package stirling.software.proprietary.cluster.valkey; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.atMost; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.data.redis.RedisSystemException; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration; +import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; + +import io.lettuce.core.RedisCommandExecutionException; +import io.lettuce.core.SslVerifyMode; + +import stirling.software.proprietary.cluster.valkey.ValkeyConnectionConfiguration.Endpoint; + +/** + * Verifies auth-fast-fail on WRONGPASS/NOAUTH/NOPERM: these are unrecoverable so the handshake must + * short-circuit after one attempt, not burn the 30 s retry loop. + */ +class ValkeyConnectionConfigurationTest { + + @Test + @DisplayName("WRONGPASS surfaces in one attempt (no 30s retry loop)") + void wrongpass_failsImmediately_withoutRetries() throws Exception { + LettuceConnectionFactory factory = mock(LettuceConnectionFactory.class); + RedisConnection conn = mock(RedisConnection.class); + when(factory.getConnection()).thenReturn(conn); + RedisCommandExecutionException auth = + new RedisCommandExecutionException("WRONGPASS invalid username-password pair"); + when(conn.ping()).thenThrow(new RedisSystemException("Error in execution", auth)); + + long start = System.nanoTime(); + IllegalStateException ex = + assertThrows( + IllegalStateException.class, + () -> + ValkeyConnectionConfiguration.eagerHandshake( + factory, "valkey", 6379, false)); + long elapsedMs = (System.nanoTime() - start) / 1_000_000; + + verify(factory, times(1)).getConnection(); + verify(conn, times(1)).ping(); + assertTrue( + elapsedMs < 1500, + "Auth failure must short-circuit retries; elapsed=" + elapsedMs + " ms"); + assertTrue( + ex.getMessage().contains("authentication failed"), + "Error message must explain the auth failure; got: " + ex.getMessage()); + verify(factory, atMost(1)).destroy(); + } + + @Test + @DisplayName("NOAUTH surfaces in one attempt") + void noauth_failsImmediately() { + LettuceConnectionFactory factory = mock(LettuceConnectionFactory.class); + RedisConnection conn = mock(RedisConnection.class); + when(factory.getConnection()).thenReturn(conn); + when(conn.ping()) + .thenThrow( + new RedisSystemException( + "Error in execution", + new RedisCommandExecutionException( + "NOAUTH Authentication required."))); + + assertThrows( + IllegalStateException.class, + () -> ValkeyConnectionConfiguration.eagerHandshake(factory, "v", 6379, false)); + verify(conn, times(1)).ping(); + } + + @Test + @DisplayName("NOPERM surfaces in one attempt") + void noperm_failsImmediately() { + LettuceConnectionFactory factory = mock(LettuceConnectionFactory.class); + RedisConnection conn = mock(RedisConnection.class); + when(factory.getConnection()).thenReturn(conn); + when(conn.ping()) + .thenThrow( + new RedisSystemException( + "Error in execution", + new RedisCommandExecutionException( + "NOPERM this user has no permissions to run the 'ping'" + + " command"))); + + assertThrows( + IllegalStateException.class, + () -> ValkeyConnectionConfiguration.eagerHandshake(factory, "v", 6379, false)); + verify(conn, times(1)).ping(); + } + + @Test + @DisplayName("isAuthFailure - direct RedisCommandExecutionException with auth prefix") + void isAuthFailure_directRedisCommandExecutionException() { + assertTrue( + ValkeyConnectionConfiguration.isAuthFailure( + new RedisCommandExecutionException("WRONGPASS bad password"))); + assertTrue( + ValkeyConnectionConfiguration.isAuthFailure( + new RedisCommandExecutionException("NOAUTH required"))); + assertTrue( + ValkeyConnectionConfiguration.isAuthFailure( + new RedisCommandExecutionException("NOPERM denied"))); + } + + @Test + @DisplayName("isAuthFailure - wrapped inside RedisSystemException (production path)") + void isAuthFailure_wrappedBySpring() { + assertTrue( + ValkeyConnectionConfiguration.isAuthFailure( + new RedisSystemException( + "Error in execution", + new RedisCommandExecutionException("WRONGPASS bad password")))); + } + + @Test + @DisplayName("isAuthFailure - connection errors do NOT count as auth failures") + void isAuthFailure_connectionErrorReturnsFalse() { + assertFalse( + ValkeyConnectionConfiguration.isAuthFailure( + new RedisSystemException( + "Redis connection failed", + new io.lettuce.core.RedisConnectionException( + "Connection refused")))); + assertFalse( + ValkeyConnectionConfiguration.isAuthFailure( + new IllegalStateException("Valkey PING returned 'foo' (expected PONG)"))); + } + + @Test + @DisplayName("bad PONG (protocol error) is not an auth failure") + void unexpectedPong_isNotAuthFailure() { + assertFalse( + ValkeyConnectionConfiguration.isAuthFailure( + new IllegalStateException("Valkey PING returned 'bar' (expected PONG)"))); + } + + @Test + @DisplayName("TLS on, skipCertVerification=false → useSsl + verifyPeer=FULL (default)") + void tls_defaultEnforcesFullPeerVerification() { + LettuceClientConfiguration cfg = + ValkeyConnectionConfiguration.buildClientConfiguration(true, false); + assertTrue(cfg.isUseSsl(), "TLS must be enabled"); + assertSame(SslVerifyMode.FULL, cfg.getVerifyMode()); + assertTrue(cfg.isVerifyPeer()); + } + + @Test + @DisplayName("TLS on, skipCertVerification=true → verifyPeer=NONE (dev override)") + void tls_skipCertVerificationOptOut() { + LettuceClientConfiguration cfg = + ValkeyConnectionConfiguration.buildClientConfiguration(true, true); + assertTrue(cfg.isUseSsl()); + assertSame(SslVerifyMode.NONE, cfg.getVerifyMode()); + } + + @Test + @DisplayName("TLS off → no SSL, verify flag default (skipCertVerification ignored)") + void noTls_ignoresSkipFlag() { + LettuceClientConfiguration cfg = + ValkeyConnectionConfiguration.buildClientConfiguration(false, true); + assertFalse(cfg.isUseSsl()); + } + + @Nested + @DisplayName("parseUrl()") + class ParseUrl { + + @Test + @DisplayName("host + explicit port, no auth, no TLS") + void hostAndPort() { + Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://valkey.internal:6380"); + assertEquals("valkey.internal", e.host()); + assertEquals(6380, e.port()); + assertFalse(e.tls()); + assertNull(e.username()); + assertNull(e.password()); + } + + @Test + @DisplayName("missing port defaults to 6379") + void defaultPort() { + assertEquals(6379, ValkeyConnectionConfiguration.parseUrl("redis://host").port()); + } + + @Test + @DisplayName("rediss:// scheme selects TLS") + void redissSelectsTls() { + assertTrue(ValkeyConnectionConfiguration.parseUrl("rediss://host:6379").tls()); + } + + @Test + @DisplayName("user:password@ sets both credentials") + void userAndPassword() { + Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://alice:s3cret@host"); + assertEquals("alice", e.username()); + assertEquals("s3cret", e.password()); + } + + @Test + @DisplayName("empty user (:pw@) is password-only auth, username stays null") + void passwordOnlyEmptyUser() { + Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://:s3cret@host"); + assertNull(e.username(), "empty user must NOT become an empty-string username"); + assertEquals("s3cret", e.password()); + } + + @Test + @DisplayName("single userinfo token (no colon) is treated as the password") + void passwordOnlyNoColon() { + Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://s3cret@host"); + assertNull(e.username()); + assertEquals("s3cret", e.password()); + } + + @Test + @DisplayName("only the first colon splits user/password (colons allowed in password)") + void colonInPassword() { + Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://user:pa:ss:word@host"); + assertEquals("user", e.username()); + assertEquals("pa:ss:word", e.password()); + } + + @Test + @DisplayName("percent-encoded reserved chars are decoded in the password") + void percentEncodedPassword() { + // %40 -> '@', %23 -> '#': both must be encoded or URI parses them structurally. + Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://:p%40ss%23word@host"); + assertNull(e.username()); + assertEquals("p@ss#word", e.password()); + } + + @Test + @DisplayName("blank url throws with a backplane-config message") + void blankUrlThrows() { + IllegalStateException ex = + assertThrows( + IllegalStateException.class, + () -> ValkeyConnectionConfiguration.parseUrl(" ")); + assertTrue(ex.getMessage().contains("cluster.valkey.url must be set")); + } + + @Test + @DisplayName("null url throws") + void nullUrlThrows() { + assertThrows( + IllegalStateException.class, + () -> ValkeyConnectionConfiguration.parseUrl(null)); + } + + @Test + @DisplayName("url with no host throws a clear error (scheme-less host:port pitfall)") + void noHostThrows() { + // "localhost:6379" parses 'localhost' as the scheme, leaving no authority/host. + IllegalStateException ex = + assertThrows( + IllegalStateException.class, + () -> ValkeyConnectionConfiguration.parseUrl("localhost:6379")); + assertTrue( + ex.getMessage().contains("has no host"), + "message must name the missing host; got: " + ex.getMessage()); + } + + @Test + @DisplayName("non-numeric port yields no host (URI registry-authority fallback)") + void nonNumericPortHasNoHost() { + // java.net.URI does not throw on a bad port; it falls back to registry authority and + // reports host=null, so this must surface as the clear no-host error, not a NPE later. + IllegalStateException ex = + assertThrows( + IllegalStateException.class, + () -> ValkeyConnectionConfiguration.parseUrl("redis://host:notaport")); + assertTrue(ex.getMessage().contains("has no host")); + } + + @Test + @DisplayName("syntactically invalid uri throws with the offending url") + void invalidUriThrows() { + IllegalStateException ex = + assertThrows( + IllegalStateException.class, + () -> ValkeyConnectionConfiguration.parseUrl("redis://ho st:6379")); + assertTrue(ex.getMessage().contains("not a valid URI")); + assertTrue(ex.getMessage().contains("redis://ho st:6379")); + } + } +}