arsvendg/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/arsvendg/Stirling-PDF.git synced 2026-07-14 10:34:06 +02:00
Code Issues Actions 4 Packages Projects Releases 1 Wiki Activity

Add Valkey cluster backplane and sticky-410 ownership (clusters) (#6472)

Browse Source
This commit is contained in:
Anthony Stirling
2026-06-02 14:59:20 +01:00
committed by GitHub
parent de9d6ad3f5
commit b355ccec9e
30 changed files with 3858 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/common/src/main/java/stirling/software/common/cluster/RateLimitStore.java
+7 -1
View File
@@ -2,7 +2,13 @@ package stirling.software.common.cluster;
import java.time.Duration;
/** Token-bucket rate limiting backed by the cluster backplane. */
/**
* Token-bucket rate limiting backed by the cluster backplane.
*
* <p>In-process implementations enforce a per-JVM limit; distributed implementations enforce a
* single global limit across every node. Both use a Bucket4j greedy-refill token bucket so the
* semantics match across single-node and cluster deployments.
*/
public interface RateLimitStore {
/**
app/common/src/main/java/stirling/software/common/service/JobExecutorService.java
+3 -132
View File
@@ -57,85 +57,38 @@ public class JobExecutorService {
this.resourceMonitor = resourceMonitor;
this.jobQueue = jobQueue;
// Parse session timeout and calculate effective timeout once during initialization
long sessionTimeoutMs = parseSessionTimeout(sessionTimeout);
this.effectiveTimeoutMs = Math.min(asyncRequestTimeoutMs, sessionTimeoutMs);
log.debug(
"Job executor configured with effective timeout of {} ms", this.effectiveTimeoutMs);
}
/**
* Run a job either asynchronously or synchronously
*
* @param async Whether to run the job asynchronously
* @param work The work to be done
* @return The response
*/
public ResponseEntity<?> runJobGeneric(boolean async, Supplier<Object> work) {
return runJobGeneric(async, work, -1);
}
/**
* Run a job either asynchronously or synchronously with a custom timeout
*
* @param async Whether to run the job asynchronously
* @param work The work to be done
* @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
* @return The response
*/
public ResponseEntity<?> runJobGeneric(
boolean async, Supplier<Object> work, long customTimeoutMs) {
return runJobGeneric(async, work, customTimeoutMs, false, 50);
}
/**
* Run a job either asynchronously or synchronously with custom parameters
*
* @param async Whether to run the job asynchronously
* @param work The work to be done
* @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
* @param queueable Whether this job can be queued when system resources are limited
* @param resourceWeight The resource weight of this job (1-100)
* @return The response
*/
public ResponseEntity<?> runJobGeneric(
boolean async,
Supplier<Object> work,
long customTimeoutMs,
boolean queueable,
int resourceWeight) {
// Generate base UUID
String baseJobId = UUID.randomUUID().toString();
// Scope job to authenticated user if security is enabled
String scopedJobKey = getScopedJobKey(baseJobId);
log.debug("Generated jobId: {} (base: {})", scopedJobKey, baseJobId);
// Store the scoped job ID in the request for potential use by other components
if (request != null) {
request.setAttribute("jobId", scopedJobKey);
// Also track this job ID in the user's session for authorization purposes
// This ensures users can only cancel their own jobs
if (request.getSession() != null) {
@SuppressWarnings("unchecked")
java.util.Set<String> userJobIds =
(java.util.Set<String>) request.getSession().getAttribute("userJobIds");
if (userJobIds == null) {
userJobIds = new java.util.concurrent.ConcurrentSkipListSet<>();
request.getSession().setAttribute("userJobIds", userJobIds);
}
userJobIds.add(scopedJobKey);
log.debug("Added scoped job ID {} to user session", scopedJobKey);
}
}
String jobId = scopedJobKey;
// Determine which timeout to use
long timeoutToUse = customTimeoutMs > 0 ? customTimeoutMs : effectiveTimeoutMs;
log.debug(
@@ -146,7 +99,6 @@ public class JobExecutorService {
queueable,
resourceWeight);
// Check if we need to queue this job based on resource availability
boolean shouldQueue =
queueable
&& async
@@ -154,7 +106,6 @@ public class JobExecutorService {
resourceMonitor.shouldQueueJob(resourceWeight);
if (shouldQueue) {
// Queue the job instead of executing immediately
log.debug(
"Queueing job {} due to resource constraints (weight: {})",
jobId,
@@ -162,18 +113,12 @@ public class JobExecutorService {
taskManager.createTask(jobId);
// Create a specialized wrapper that updates the TaskManager
final String capturedJobIdForQueue = jobId;
Supplier<Object> wrappedWork =
() -> {
try {
// Set jobId in ThreadLocal context for the queued job
stirling.software.common.util.JobContext.setJobId(
capturedJobIdForQueue);
log.debug(
"Set jobId {} in JobContext for queued job execution",
capturedJobIdForQueue);
Object result = work.get();
processJobResult(capturedJobIdForQueue, result);
return result;
@@ -186,21 +131,17 @@ public class JobExecutorService {
taskManager.setError(capturedJobIdForQueue, e.getMessage());
throw e;
} finally {
// Clean up ThreadLocal to avoid memory leaks
stirling.software.common.util.JobContext.clear();
}
};
// Queue the job and get the future
CompletableFuture<ResponseEntity<?>> future =
jobQueue.queueJob(jobId, resourceWeight, wrappedWork, timeoutToUse);
// Return immediately with job ID
return ResponseEntity.ok().body(new JobResponse<>(true, jobId, null));
} else if (async) {
taskManager.createTask(jobId);
// Capture the jobId for the async thread
final String capturedJobId = jobId;
executor.execute(
@@ -211,13 +152,7 @@ public class JobExecutorService {
capturedJobId,
timeoutToUse);
// Set jobId in ThreadLocal context for the async thread
stirling.software.common.util.JobContext.setJobId(capturedJobId);
log.debug(
"Set jobId {} in JobContext for async execution",
capturedJobId);
// Execute with timeout
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
processJobResult(capturedJobId, result);
} catch (TimeoutException te) {
@@ -227,7 +162,6 @@ public class JobExecutorService {
log.error("Error executing job {}: {}", jobId, e.getMessage(), e);
taskManager.setError(jobId, e.getMessage());
} finally {
// Clean up ThreadLocal to avoid memory leaks
stirling.software.common.util.JobContext.clear();
}
});
@@ -237,27 +171,19 @@ public class JobExecutorService {
try {
log.debug("Running sync job with timeout {} ms", timeoutToUse);
// Make jobId available to downstream components on the worker thread
stirling.software.common.util.JobContext.setJobId(jobId);
log.debug("Set jobId {} in JobContext for sync execution", jobId);
// Execute with timeout
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
// If the result is already a ResponseEntity, return it directly
if (result instanceof ResponseEntity) {
return (ResponseEntity<?>) result;
}
// Process different result types
return handleResultForSyncJob(result);
} catch (TimeoutException te) {
log.error("Synchronous job timed out after {} ms", timeoutToUse);
return ResponseEntity.internalServerError()
.body(Map.of("error", "Job timed out after " + timeoutToUse + " ms"));
} catch (RuntimeException e) {
// Check if this is a typed exception that should be handled by
// GlobalExceptionHandler (either directly or wrapped)
Throwable cause = e.getCause();
if (e instanceof IllegalArgumentException
|| cause
@@ -267,16 +193,13 @@ public class JobExecutorService {
instanceof
stirling.software.common.util.ExceptionUtils
.BaseValidationException) {
// Rethrow so GlobalExceptionHandler can handle with proper HTTP status codes
throw e;
}
// Handle other RuntimeExceptions as generic errors
log.error("Error executing synchronous job: {}", e.getMessage(), e);
return ResponseEntity.internalServerError()
.body(Map.of("error", "Job failed: " + e.getMessage()));
} catch (Exception e) {
log.error("Error executing synchronous job: {}", e.getMessage(), e);
// Construct a JSON error response
return ResponseEntity.internalServerError()
.body(Map.of("error", "Job failed: " + e.getMessage()));
} finally {
@@ -285,23 +208,13 @@ public class JobExecutorService {
}
}
/**
* Process the result of an asynchronous job
*
* @param jobId The job ID
* @param result The result
*/
private void processJobResult(String jobId, Object result) {
try {
if (result instanceof byte[]) {
// Store byte array directly to disk to avoid double memory consumption
String fileId = fileStorage.storeBytes((byte[]) result, "result.pdf");
taskManager.setFileResult(
jobId, fileId, "result.pdf", MediaType.APPLICATION_PDF_VALUE);
log.debug("Stored byte[] result with fileId: {}", fileId);
// Let the byte array get collected naturally in the next GC cycle
// We don't need to force System.gc() which can be harmful
} else if (result instanceof ResponseEntity) {
ResponseEntity<?> response = (ResponseEntity<?>) result;
Object body = response.getBody();
@@ -330,16 +243,13 @@ public class JobExecutorService {
taskManager.setFileResult(jobId, fileId, filename, contentType);
log.debug("Stored ResponseEntity<Resource> result with fileId: {}", fileId);
} else {
// Check if the response body contains a fileId
if (body != null && body.toString().contains("fileId")) {
try {
// Try to extract fileId using reflection
java.lang.reflect.Method getFileId =
body.getClass().getMethod("getFileId");
String fileId = (String) getFileId.invoke(body);
if (fileId != null && !fileId.isEmpty()) {
// Try to get filename and content type
String filename = "result.pdf";
String contentType = MediaType.APPLICATION_PDF_VALUE;
@@ -379,7 +289,6 @@ public class JobExecutorService {
}
}
// Store generic result
taskManager.setResult(jobId, body);
}
} else if (result instanceof MultipartFile file) {
@@ -388,16 +297,13 @@ public class JobExecutorService {
jobId, fileId, file.getOriginalFilename(), file.getContentType());
log.debug("Stored MultipartFile result with fileId: {}", fileId);
} else {
// Check if result has a fileId field
if (result != null) {
try {
// Try to extract fileId using reflection
java.lang.reflect.Method getFileId =
result.getClass().getMethod("getFileId");
String fileId = (String) getFileId.invoke(result);
if (fileId != null && !fileId.isEmpty()) {
// Try to get filename and content type
String filename = "result.pdf";
String contentType = MediaType.APPLICATION_PDF_VALUE;
@@ -435,7 +341,6 @@ public class JobExecutorService {
}
}
// Default case: store the result as is
taskManager.setResult(jobId, result);
}
@@ -446,16 +351,8 @@ public class JobExecutorService {
}
}
/**
* Handle different result types for synchronous jobs
*
* @param result The result object
* @return The appropriate ResponseEntity
* @throws IOException If there is an error processing the result
*/
private ResponseEntity<?> handleResultForSyncJob(Object result) throws IOException {
if (result instanceof byte[]) {
// Return byte array as PDF
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_PDF)
.header(
@@ -463,7 +360,6 @@ public class JobExecutorService {
"form-data; name=\"attachment\"; filename=\"result.pdf\"")
.body(result);
} else if (result instanceof MultipartFile file) {
// Return MultipartFile content
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(file.getContentType()))
.header(
@@ -473,7 +369,6 @@ public class JobExecutorService {
+ "\"")
.body(file.getBytes());
} else {
// Default case: return as JSON
return ResponseEntity.ok(result);
}
}
@@ -493,15 +388,9 @@ public class JobExecutorService {
return mediaType != null ? mediaType.toString() : MediaType.APPLICATION_PDF_VALUE;
}
/**
* Parse session timeout string (e.g., "30m", "1h") to milliseconds
*
* @param timeout The timeout string
* @return The timeout in milliseconds
*/
private long parseSessionTimeout(String timeout) {
if (timeout == null || timeout.isEmpty()) {
return 30 * 60 * 1000; // Default: 30 minutes
return 30 * 60 * 1000;
}
try {
@@ -523,27 +412,16 @@ public class JobExecutorService {
case "m" -> (long) (numericValue * 60 * 1000);
case "h" -> (long) (numericValue * 60 * 60 * 1000);
case "d" -> (long) (numericValue * 24 * 60 * 60 * 1000);
default -> (long) (numericValue * 60 * 1000); // Default to minutes
default -> (long) (numericValue * 60 * 1000);
};
} catch (Exception e) {
log.warn("Could not parse session timeout '{}', using default", timeout);
return 30 * 60 * 1000; // Default: 30 minutes
return 30 * 60 * 1000;
}
}
/**
* Execute a supplier with a timeout
*
* @param supplier The supplier to execute
* @param timeoutMs The timeout in milliseconds
* @return The result from the supplier
* @throws TimeoutException If the execution times out
* @throws Exception If the supplier throws an exception
*/
private <T> T executeWithTimeout(Supplier<T> supplier, long timeoutMs)
throws TimeoutException, Exception {
// Use the same executor as other async jobs for consistency
// This ensures all operations run on the same thread pool
String currentJobId = stirling.software.common.util.JobContext.getJobId();
java.util.concurrent.CompletableFuture<T> future =
@@ -577,17 +455,10 @@ public class JobExecutorService {
}
}
/**
* Get a scoped job key that includes user ownership when security is enabled.
*
* @param baseJobId the base job identifier
* @return scoped job key, or just baseJobId if no ownership service available
*/
private String getScopedJobKey(String baseJobId) {
if (jobOwnershipService != null) {
return jobOwnershipService.createScopedJobKey(baseJobId);
}
// Security disabled, return unsecured job key
return baseJobId;
}
}
app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessDistributedLockTest.java
+3 -1
View File
@@ -24,7 +24,9 @@ class InProcessDistributedLockTest {
}
@Test
void reentryFromSameThreadFails() {
void reentryFromSameThreadFails_parityWithValkey() {
// The Valkey impl refuses reentry (SET NX semantics); the in-process impl must match,
// otherwise code working in single-instance silently breaks in cluster mode.
DistributedLock lock = new InProcessDistributedLock();
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
Optional<DistributedLock.LockHandle> reentry = lock.tryAcquire("k", Duration.ofSeconds(30));
app/common/src/test/java/stirling/software/common/service/TaskManagerJobStoreDelegationTest.java
+2
View File
@@ -86,6 +86,8 @@ class TaskManagerJobStoreDelegationTest {
@Override
public boolean shouldRunLocalCleanup() {
// Distributed backplanes own job TTL eviction themselves; this mock
// mirrors the real ValkeyClusterBackplane override of the default true.
return false;
}
};
app/core/src/main/java/stirling/software/common/controller/JobController.java
+142 -77
View File
@@ -4,6 +4,7 @@ import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
@@ -22,6 +23,10 @@ import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.ClusterBackplane;
import stirling.software.common.cluster.JobStore;
import stirling.software.common.cluster.JobStoreEntry;
import stirling.software.common.cluster.StickyMissRecorder;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.model.job.ResultFile;
import stirling.software.common.service.FileStorage;
@@ -30,7 +35,6 @@ import stirling.software.common.service.JobQueue;
import stirling.software.common.service.TaskManager;
import stirling.software.common.util.RegexPatternUtils;
/** REST controller for job-related endpoints */
@RestController
@RequiredArgsConstructor
@Slf4j
@@ -42,20 +46,29 @@ public class JobController {
private final FileStorage fileStorage;
private final JobQueue jobQueue;
private final HttpServletRequest request;
private final ClusterBackplane clusterBackplane;
private final JobStore jobStore;
// Short-TTL local cache fronting JobStore.get() on the sticky-410 path to avoid a Valkey
// HGETALL round-trip on every download retry for the same job.
private final JobOwnershipCache ownershipCache = new JobOwnershipCache();
@Autowired(required = false)
private JobOwnershipService jobOwnershipService;
/**
* Get the status of a job
*
* @param jobId The job ID
* @return The job result
*/
@Autowired(required = false)
private StickyMissRecorder stickyMissRecorder;
@GetMapping("/job/{jobId}")
@Operation(summary = "Get job status")
public ResponseEntity<?> getJobStatus(@PathVariable("jobId") String jobId) {
// Validate job ownership
// Sticky-410 must run before user-auth: a 403 here would leak job existence and defeat
// LB re-routing. The owner node is where the real auth check should happen.
Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
if (peerOwned.isPresent()) {
return peerOwned.get();
}
if (!validateJobAccess(jobId)) {
log.warn("Unauthorized attempt to access job status: {}", jobId);
return ResponseEntity.status(403)
@@ -67,7 +80,6 @@ public class JobController {
return ResponseEntity.notFound().build();
}
// Check if the job is in the queue and add queue information
if (!result.isComplete() && jobQueue.isJobQueued(jobId)) {
int position = jobQueue.getJobPosition(jobId);
Map<String, Object> resultWithQueueInfo =
@@ -82,16 +94,14 @@ public class JobController {
return ResponseEntity.ok(result);
}
/**
* Get the result of a job
*
* @param jobId The job ID
* @return The job result
*/
@GetMapping("/job/{jobId}/result")
@Operation(summary = "Get job result")
public ResponseEntity<?> getJobResult(@PathVariable("jobId") String jobId) {
// Validate job ownership
Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
if (peerOwned.isPresent()) {
return peerOwned.get();
}
if (!validateJobAccess(jobId)) {
log.warn("Unauthorized attempt to access job result: {}", jobId);
return ResponseEntity.status(403)
@@ -111,7 +121,6 @@ public class JobController {
return ResponseEntity.badRequest().body("Job failed: " + result.getError());
}
// Handle multiple files - return metadata for client to download individually
if (result.hasMultipleFiles()) {
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_JSON)
@@ -125,11 +134,11 @@ public class JobController {
result.getAllResultFiles()));
}
// Handle single file (download directly)
if (result.hasFiles() && !result.hasMultipleFiles()) {
try {
List<ResultFile> files = result.getAllResultFiles();
ResultFile singleFile = files.get(0);
byte[] fileContent = fileStorage.retrieveBytes(singleFile.getFileId());
return ResponseEntity.ok()
.header("Content-Type", singleFile.getContentType())
@@ -147,30 +156,22 @@ public class JobController {
return ResponseEntity.ok(result.getResult());
}
// Admin-only endpoints have been moved to AdminJobController in the proprietary package
/**
* Cancel a job by its ID
*
* <p>This method should only allow cancellation of jobs that were created by the current user.
* The jobId should be part of the user's session or otherwise linked to their identity.
*
* @param jobId The job ID
* @return Response indicating whether the job was cancelled
*/
@DeleteMapping("/job/{jobId}")
@Operation(summary = "Cancel a job")
public ResponseEntity<?> cancelJob(@PathVariable("jobId") String jobId) {
log.debug("Request to cancel job: {}", jobId);
// Validate job ownership
Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
if (peerOwned.isPresent()) {
return peerOwned.get();
}
if (!validateJobAccess(jobId)) {
log.warn("Unauthorized attempt to cancel job: {}", jobId);
return ResponseEntity.status(403)
.body(Map.of("message", "You are not authorized to cancel this job"));
}
// First check if the job is in the queue
boolean cancelled = false;
int queuePosition = -1;
@@ -180,11 +181,9 @@ public class JobController {
log.info("Cancelled queued job: {} (was at position {})", jobId, queuePosition);
}
// If not in queue or couldn't cancel, try to cancel in TaskManager
if (!cancelled) {
JobResult result = taskManager.getJobResult(jobId);
if (result != null && !result.isComplete()) {
// Mark as error with cancellation message
taskManager.setError(jobId, "Job was cancelled by user");
cancelled = true;
log.info("Marked job as cancelled in TaskManager: {}", jobId);
@@ -201,7 +200,6 @@ public class JobController {
"queuePosition",
queuePosition >= 0 ? queuePosition : "n/a"));
} else {
// Job not found or already complete
JobResult result = taskManager.getJobResult(jobId);
if (result == null) {
return ResponseEntity.notFound().build();
@@ -215,16 +213,14 @@ public class JobController {
}
}
/**
* Get the list of files for a job
*
* @param jobId The job ID
* @return List of files for the job
*/
@GetMapping("/job/{jobId}/result/files")
@Operation(summary = "Get job result files")
public ResponseEntity<?> getJobFiles(@PathVariable("jobId") String jobId) {
// Validate job ownership
Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
if (peerOwned.isPresent()) {
return peerOwned.get();
}
if (!validateJobAccess(jobId)) {
log.warn("Unauthorized attempt to access job files: {}", jobId);
return ResponseEntity.status(403)
@@ -252,28 +248,31 @@ public class JobController {
"files", files));
}
/**
* Get metadata for an individual file by its file ID
*
* @param fileId The file ID
* @return The file metadata
*/
@GetMapping("/files/{fileId}/metadata")
@Operation(summary = "Get file metadata")
public ResponseEntity<?> getFileMetadata(@PathVariable("fileId") String fileId) {
try {
String jobKey = taskManager.findJobKeyByFileId(fileId);
String jobKey;
try {
jobKey = taskManager.findJobKeyByFileId(fileId);
} catch (RuntimeException backplaneEx) {
return backplaneUnavailable(fileId, backplaneEx);
}
if (jobKey == null) {
return ResponseEntity.notFound().build();
}
Optional<ResponseEntity<?>> notOwner = guardNonOwner(jobKey);
if (notOwner.isPresent()) {
return notOwner.get();
}
if (!validateJobAccess(jobKey)) {
log.warn("Unauthorized attempt to access file metadata: {}", fileId);
return ResponseEntity.status(403)
.body(Map.of("message", "You are not authorized to access this file"));
}
// Find the file metadata from any job that contains this file
ResultFile resultFile = taskManager.findResultFileByFileId(fileId);
if (resultFile != null) {
@@ -281,12 +280,10 @@ public class JobController {
}
if (!isSecurityEnabled()) {
// Backwards compatibility when ownership service is unavailable
if (!fileStorage.fileExists(fileId)) {
return ResponseEntity.notFound().build();
}
// File exists but no metadata found, get basic info efficiently
long fileSize = fileStorage.getFileSize(fileId);
return ResponseEntity.ok(
Map.of(
@@ -308,32 +305,31 @@ public class JobController {
}
}
/**
* Download an individual file by its file ID
*
* @param fileId The file ID
* @return The file content
*/
@GetMapping("/files/{fileId}")
@Operation(summary = "Download a file")
public ResponseEntity<?> downloadFile(@PathVariable("fileId") String fileId) {
try {
String jobKey = taskManager.findJobKeyByFileId(fileId);
String jobKey;
try {
jobKey = taskManager.findJobKeyByFileId(fileId);
} catch (RuntimeException backplaneEx) {
return backplaneUnavailable(fileId, backplaneEx);
}
if (jobKey == null) {
return ResponseEntity.notFound().build();
}
Optional<ResponseEntity<?>> notOwner = guardNonOwner(jobKey);
if (notOwner.isPresent()) {
return notOwner.get();
}
if (!validateJobAccess(jobKey)) {
log.warn("Unauthorized attempt to download file: {}", fileId);
return ResponseEntity.status(403)
.body(Map.of("message", "You are not authorized to access this file"));
}
// Retrieve file content
byte[] fileContent = fileStorage.retrieveBytes(fileId);
// Find the file metadata from any job that contains this file
// This is for getting the original filename and content type
ResultFile resultFile = taskManager.findResultFileByFileId(fileId);
String fileName = resultFile != null ? resultFile.getFileName() : "download";
@@ -342,6 +338,8 @@ public class JobController {
? resultFile.getContentType()
: MediaType.APPLICATION_OCTET_STREAM_VALUE;
byte[] fileContent = fileStorage.retrieveBytes(fileId);
return ResponseEntity.ok()
.header("Content-Type", contentType)
.header("Content-Disposition", createContentDispositionHeader(fileName))
@@ -357,11 +355,88 @@ public class JobController {
}
/**
* Create Content-Disposition header with UTF-8 filename support
*
* @param fileName The filename to encode
* @return Content-Disposition header value
* Returns 410 Gone when the job is owned by a peer node, empty otherwise. Uses a short-TTL
* local cache to avoid repeated Valkey lookups on the hot download path. When the backplane is
* unreachable, a locally-held job is still served and anything else gets a retryable 503.
*/
private Optional<ResponseEntity<?>> guardNonOwner(String jobId) {
if (clusterBackplane == null || jobStore == null) {
return Optional.empty();
}
Optional<JobStoreEntry> entry;
Optional<Optional<JobStoreEntry>> cached = ownershipCache.get(jobId);
if (cached.isPresent()) {
entry = cached.get();
} else {
try {
entry = jobStore.get(jobId);
} catch (RuntimeException ex) {
// Backplane unreachable: if we hold the job locally serve it, otherwise return a
// retryable 503 (same contract as the file endpoints) instead of a misleading 404.
if (taskManager.getJobResult(jobId) == null) {
return Optional.of(backplaneUnavailable(jobId, ex));
}
log.warn(
"JobStore lookup failed for jobId={}; serving locally-held job: {}",
jobId,
ex.getMessage());
return Optional.empty();
}
ownershipCache.put(jobId, entry);
}
if (entry.isEmpty()) {
return Optional.empty();
}
String owner = entry.get().owningNodeId();
if (owner == null || owner.isBlank()) {
return Optional.empty();
}
String localId = clusterBackplane.localNodeId();
if (owner.equals(localId)) {
return Optional.empty();
}
log.info(
"Sticky-session miss for jobId={} (owner={}, local={}); returning 410 so client"
+ " retries via LB affinity",
jobId,
owner,
localId);
if (stickyMissRecorder != null) {
stickyMissRecorder.recordStickyMiss();
}
return Optional.of(
ResponseEntity.status(410)
.header("Retry-After", "0")
.body(
Map.of(
"message",
"Result lives on another node. Retry to be routed there"
+ " by the load balancer's sticky-session"
+ " affinity, or re-run the job.",
"ownedBy",
owner,
"currentNode",
localId == null ? "" : localId)));
}
/**
* When the backplane is unreachable we cannot resolve ownership or existence, and serving
* without that check would be unsafe - so return a retryable 503 (consistent with the
* sticky-410 retry model) rather than a misleading 404 or a generic 500.
*/
private ResponseEntity<?> backplaneUnavailable(String id, RuntimeException ex) {
log.warn(
"Backplane lookup failed for {}; returning 503 (retryable): {}",
id,
ex.getMessage());
return ResponseEntity.status(503)
.header("Retry-After", "1")
.body(
Map.of(
"message",
"Cluster backplane temporarily unavailable; retry shortly."));
}
private String createContentDispositionHeader(String fileName) {
try {
String encodedFileName =
@@ -371,19 +446,11 @@ public class JobController {
.replaceAll("%20"); // URLEncoder uses + for spaces, but we want %20
return "attachment; filename=\"" + fileName + "\"; filename*=UTF-8''" + encodedFileName;
} catch (Exception e) {
// Fallback to basic filename if encoding fails
return "attachment; filename=\"" + fileName + "\"";
}
}
/**
* Validate that the current user has access to the given job.
*
* @param jobId the job identifier to validate
* @return true if user has access, false otherwise
*/
private boolean validateJobAccess(String jobId) {
// If JobOwnershipService is available (security enabled), use it
if (jobOwnershipService != null) {
try {
return jobOwnershipService.validateJobAccess(jobId);
@@ -393,8 +460,6 @@ public class JobController {
}
}
// Security disabled - allow all access (backwards compatibility)
// When security is not enabled, any user can access any job by jobId
return true;
}
}
app/core/src/main/java/stirling/software/common/controller/JobOwnershipCache.java
+47
View File
@@ -0,0 +1,47 @@
package stirling.software.common.controller;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import stirling.software.common.cluster.JobStoreEntry;
/**
* Process-local TTL cache for {@link JobStoreEntry} lookups to suppress redundant Valkey HGETALL
* round-trips on the hot result-download path (sticky-410 ownership check).
*
* <p>5 second TTL is short enough that a job's lifecycle transitions (RUNNING -> COMPLETE -> TTL
* expiry) propagate to all nodes within the LB's sticky-session window, and short enough that a
* mistakenly-cached "not found" recovers quickly when an entry actually shows up. Cap the map at
* 2048 entries to bound memory; eviction is best-effort (clear-and-restart) since the cache is
* advisory.
*/
final class JobOwnershipCache {
private static final long TTL_NANOS = 5L * 1_000_000_000L; // 5 s
private static final int MAX_ENTRIES = 2048;
private final ConcurrentMap<String, Entry> entries = new ConcurrentHashMap<>();
Optional<Optional<JobStoreEntry>> get(String jobId) {
Entry e = entries.get(jobId);
if (e == null) {
return Optional.empty();
}
if (System.nanoTime() - e.storedAtNanos > TTL_NANOS) {
entries.remove(jobId, e);
return Optional.empty();
}
return Optional.of(e.value);
}
void put(String jobId, Optional<JobStoreEntry> value) {
if (entries.size() >= MAX_ENTRIES) {
// Best-effort eviction; under burst the cache simply rebuilds.
entries.clear();
}
entries.put(jobId, new Entry(value, System.nanoTime()));
}
private record Entry(Optional<JobStoreEntry> value, long storedAtNanos) {}
}
app/core/src/test/java/stirling/software/common/controller/JobControllerOwnershipTest.java
+479
View File
@@ -0,0 +1,479 @@
package stirling.software.common.controller;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.util.ReflectionTestUtils;
import jakarta.servlet.http.HttpServletRequest;
import stirling.software.common.cluster.ClusterBackplane;
import stirling.software.common.cluster.JobStore;
import stirling.software.common.cluster.JobStoreEntry;
import stirling.software.common.cluster.StickyMissRecorder;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.JobOwnershipService;
import stirling.software.common.service.JobQueue;
import stirling.software.common.service.TaskManager;
/**
* Sticky-410 ownership contract for {@link JobController}: peer-owned jobs return 410 Gone with
* ownedBy/currentNode fields; locally-owned and no-entry cases return 200. FileStorage is never
* touched on the 410 path. Manual mock construction so tests can vary backplane/jobstore combos.
*/
class JobControllerOwnershipTest {
private TaskManager taskManager;
private FileStorage fileStorage;
private JobQueue jobQueue;
private HttpServletRequest request;
private JobOwnershipService jobOwnershipService;
private ClusterBackplane clusterBackplane;
private JobStore jobStore;
private StickyMissRecorder stickyMissRecorder;
private static final String JOB_ID = "job-42";
private static final String FILE_ID = "file-abc";
private static final String LOCAL_NODE = "node-self";
private static final String PEER_NODE = "node-peer";
@BeforeEach
void setUp() {
taskManager = mock(TaskManager.class);
fileStorage = mock(FileStorage.class);
jobQueue = mock(JobQueue.class);
request = mock(HttpServletRequest.class);
jobOwnershipService = mock(JobOwnershipService.class);
clusterBackplane = mock(ClusterBackplane.class);
jobStore = mock(JobStore.class);
stickyMissRecorder = mock(StickyMissRecorder.class);
}
private JobController makeController(ClusterBackplane backplane, JobStore store) {
JobController c =
new JobController(taskManager, fileStorage, jobQueue, request, backplane, store);
ReflectionTestUtils.setField(c, "stickyMissRecorder", stickyMissRecorder);
return c;
}
private JobController makeController() {
return makeController(clusterBackplane, jobStore);
}
private JobStoreEntry entryOwnedBy(String ownerNodeId) {
return new JobStoreEntry(
JOB_ID,
JobStoreEntry.JobState.COMPLETE,
ownerNodeId,
Instant.now(),
Instant.now(),
null,
List.of(FILE_ID),
Map.of());
}
private JobResult completedJobWithFile() {
JobResult result = new JobResult();
result.setJobId(JOB_ID);
// completeWithSingleFile populates the resultFiles list, sets complete=true,
// and sets completedAt - all required for the getJobResult single-file branch.
result.completeWithSingleFile(FILE_ID, "out.pdf", "application/pdf", 7L);
return result;
}
@Test
@DisplayName(
"downloadFile peer-owned → full sticky-410 contract"
+ " (status + Retry-After + payload + metric + storage untouched)")
void downloadFile_peerOwned_fullStickyContract() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
assertEquals(HttpStatus.GONE, response.getStatusCode());
assertEquals("0", response.getHeaders().getFirst("Retry-After"));
assertInstanceOf(Map.class, response.getBody());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals(3, body.size(), "exactly: message, ownedBy, currentNode");
assertEquals(PEER_NODE, body.get("ownedBy"));
assertEquals(LOCAL_NODE, body.get("currentNode"));
assertNotNull(body.get("message"));
assertTrue(((String) body.get("message")).toLowerCase().contains("retry"));
assertNull(body.get("internalSecret"));
assertNull(body.get("filePath"));
verify(stickyMissRecorder).recordStickyMiss();
verify(fileStorage, never()).retrieveBytes(FILE_ID);
}
private static Stream<Arguments> downloadHappyPathScenarios() {
return Stream.of(
Arguments.of("locallyOwned", LOCAL_NODE, true),
Arguments.of("noJobStoreEntry", null, false),
Arguments.of("blankOwningNodeId", "", true));
}
@ParameterizedTest(name = "downloadFile {0} -> 200, no sticky-miss")
@MethodSource("downloadHappyPathScenarios")
void downloadFile_happyPath_returnsOkAndNoMetric(
String scenario, String ownerNodeId, boolean entryPresent) throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID))
.thenReturn(
entryPresent ? Optional.of(entryOwnedBy(ownerNodeId)) : Optional.empty());
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
assertEquals(HttpStatus.OK, response.getStatusCode(), scenario);
verify(fileStorage).retrieveBytes(FILE_ID);
verify(stickyMissRecorder, never()).recordStickyMiss();
}
@Test
@DisplayName("getJobResult: locally-owned single-file result → reads from FileStorage, 200 OK")
void getJobResult_singleFile_locallyOwned_readsFromStorage() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
ResponseEntity<?> response = makeController().getJobResult(JOB_ID);
assertEquals(HttpStatus.OK, response.getStatusCode());
}
private enum Endpoint {
DOWNLOAD_FILE,
GET_JOB_RESULT,
GET_JOB_STATUS,
GET_JOB_FILES,
GET_FILE_METADATA,
CANCEL_JOB
}
private static Stream<Arguments> peerOwned410Scenarios() {
return Stream.of(
Arguments.of(Endpoint.DOWNLOAD_FILE),
Arguments.of(Endpoint.GET_JOB_RESULT),
Arguments.of(Endpoint.GET_JOB_STATUS),
Arguments.of(Endpoint.GET_JOB_FILES),
Arguments.of(Endpoint.GET_FILE_METADATA),
Arguments.of(Endpoint.CANCEL_JOB));
}
@ParameterizedTest(name = "{0} peer-owned -> 410, ownedBy=peer, metric++")
@MethodSource("peerOwned410Scenarios")
void endpoint_peerOwned_returns410(Endpoint endpoint) throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
switch (endpoint) {
case DOWNLOAD_FILE, GET_FILE_METADATA ->
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
case GET_JOB_RESULT ->
when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
case GET_JOB_STATUS, GET_JOB_FILES ->
when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
case CANCEL_JOB -> {
when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
}
}
ResponseEntity<?> response =
switch (endpoint) {
case DOWNLOAD_FILE -> makeController().downloadFile(FILE_ID);
case GET_JOB_RESULT -> makeController().getJobResult(JOB_ID);
case GET_JOB_STATUS -> makeController().getJobStatus(JOB_ID);
case GET_JOB_FILES -> makeController().getJobFiles(JOB_ID);
case GET_FILE_METADATA -> makeController().getFileMetadata(FILE_ID);
case CANCEL_JOB -> makeController().cancelJob(JOB_ID);
};
assertEquals(HttpStatus.GONE, response.getStatusCode());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals(PEER_NODE, body.get("ownedBy"));
assertEquals(LOCAL_NODE, body.get("currentNode"));
verify(stickyMissRecorder).recordStickyMiss();
verify(fileStorage, never()).retrieveBytes(FILE_ID);
if (endpoint == Endpoint.CANCEL_JOB) {
verify(taskManager, never()).setError(JOB_ID, "Job was cancelled by user");
}
}
private static Stream<Arguments> unknownJob404Scenarios() {
return Stream.of(Arguments.of(Endpoint.GET_JOB_STATUS), Arguments.of(Endpoint.CANCEL_JOB));
}
@ParameterizedTest(name = "{0} unknown jobId -> 404 (not 410), no metric")
@MethodSource("unknownJob404Scenarios")
void endpoint_unknownJob_returns404(Endpoint endpoint) {
when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
when(jobStore.get(JOB_ID)).thenReturn(Optional.empty());
if (endpoint == Endpoint.CANCEL_JOB) {
when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
}
ResponseEntity<?> response =
switch (endpoint) {
case GET_JOB_STATUS -> makeController().getJobStatus(JOB_ID);
case CANCEL_JOB -> makeController().cancelJob(JOB_ID);
default -> throw new IllegalArgumentException(endpoint.name());
};
assertEquals(HttpStatus.NOT_FOUND, response.getStatusCode());
verify(stickyMissRecorder, never()).recordStickyMiss();
}
@Test
@DisplayName("Single-instance install (no ClusterBackplane bean): no 410, no NPE")
void singleInstance_noClusterBackplane_noGoneResponse() throws Exception {
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
ResponseEntity<?> response = makeController(null, jobStore).downloadFile(FILE_ID);
assertEquals(HttpStatus.OK, response.getStatusCode());
verify(fileStorage).retrieveBytes(FILE_ID);
}
@Test
@DisplayName("Single-instance install (no JobStore bean): no 410, no NPE")
void singleInstance_noJobStore_noGoneResponse() throws Exception {
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
ResponseEntity<?> response = makeController(clusterBackplane, null).downloadFile(FILE_ID);
assertEquals(HttpStatus.OK, response.getStatusCode());
verify(fileStorage).retrieveBytes(FILE_ID);
}
@Test
@DisplayName("Single-instance (no StickyMissRecorder bean) → no NPE, still 200 OK")
void noStickyMissRecorder_works() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
JobController c = makeController();
ReflectionTestUtils.setField(c, "stickyMissRecorder", null);
ResponseEntity<?> response = c.downloadFile(FILE_ID);
assertEquals(HttpStatus.OK, response.getStatusCode());
}
@Test
@DisplayName(
"cluster-mode but localNodeId is null → no NPE; 410 because owner is set and"
+ " differs from blank")
void clusterBackplanePresent_butLocalNodeIdNull_falsBackGracefully() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(null);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
// We still 410: owner is "node-peer", local is null → they don't match. Rather than
// silently 200-from-wrong-disk (which would serve garbage), we surface the mismatch.
ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
assertEquals(HttpStatus.GONE, response.getStatusCode());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals("", body.get("currentNode"), "blank when localNodeId is null");
assertEquals(PEER_NODE, body.get("ownedBy"));
}
@Test
@DisplayName("Owner returns 410 even when JobOwnershipService allows access (orthogonal)")
void ownershipService_passes_butStickyStillReturns410() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(true);
JobController c = makeController();
ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
ResponseEntity<?> response = c.downloadFile(FILE_ID);
assertEquals(HttpStatus.GONE, response.getStatusCode());
}
@Test
@DisplayName(
"downloadFile: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak"
+ " file existence")
void downloadFile_peerOwned_ownershipDenied_returns410NotForbidden() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
JobController c = makeController();
ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
ResponseEntity<?> response = c.downloadFile(FILE_ID);
assertEquals(HttpStatus.GONE, response.getStatusCode());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals(PEER_NODE, body.get("ownedBy"));
verify(fileStorage, never()).retrieveBytes(FILE_ID);
}
@Test
@DisplayName(
"getJobStatus: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak"
+ " job existence")
void getJobStatus_peerOwned_ownershipDenied_returns410NotForbidden() {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
JobController c = makeController();
ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
ResponseEntity<?> response = c.getJobStatus(JOB_ID);
assertEquals(HttpStatus.GONE, response.getStatusCode());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals(PEER_NODE, body.get("ownedBy"));
}
@Test
@DisplayName(
"cancelJob: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak job"
+ " existence")
void cancelJob_peerOwned_ownershipDenied_returns410NotForbidden() {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
JobController c = makeController();
ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
ResponseEntity<?> response = c.cancelJob(JOB_ID);
assertEquals(HttpStatus.GONE, response.getStatusCode());
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertEquals(PEER_NODE, body.get("ownedBy"));
verify(taskManager, never()).setError(JOB_ID, "Job was cancelled by user");
}
@Test
@DisplayName(
"guardNonOwner caches JobStore.get within TTL window: second call same jobId hits"
+ " cache, not Valkey")
void guardNonOwner_cachesJobStoreLookupWithinTtl() throws Exception {
when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
JobController c = makeController();
c.downloadFile(FILE_ID);
c.downloadFile(FILE_ID);
c.downloadFile(FILE_ID);
verify(jobStore, times(1)).get(JOB_ID);
}
@Test
@DisplayName(
"guardNonOwner: JobStore.get throws (Valkey timeout) → falls through to local-disk"
+ " path, no 500 leaks to caller")
void guardNonOwner_jobStoreException_fallsThroughToLocalPath() throws Exception {
when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout"));
when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
assertEquals(HttpStatus.OK, response.getStatusCode());
verify(fileStorage).retrieveBytes(FILE_ID);
verify(stickyMissRecorder, never()).recordStickyMiss();
}
@Test
@DisplayName("backplane down + job NOT held locally → 503 retryable (not a misleading 404)")
void jobEndpoint_backplaneDown_notLocal_returns503() {
when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout"));
when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
ResponseEntity<?> response = makeController().getJobStatus(JOB_ID);
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
assertEquals("1", response.getHeaders().getFirst("Retry-After"));
verify(stickyMissRecorder, never()).recordStickyMiss();
}
@Test
@DisplayName("backplane down but job held locally → owner still serves (not 503)")
void jobEndpoint_backplaneDown_local_servesLocally() {
when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout"));
when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
ResponseEntity<?> response = makeController().getJobStatus(JOB_ID);
assertEquals(HttpStatus.OK, response.getStatusCode());
}
@Test
@DisplayName(
"downloadFile: findJobKeyByFileId throws (backplane down) → 503 + Retry-After,"
+ " not 404/500, storage untouched")
void downloadFile_findJobKeyThrows_returns503Retryable() throws Exception {
when(taskManager.findJobKeyByFileId(FILE_ID))
.thenThrow(new RuntimeException("Valkey command timeout"));
ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
assertEquals("1", response.getHeaders().getFirst("Retry-After"));
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertTrue(((String) body.get("message")).toLowerCase().contains("unavailable"));
verify(fileStorage, never()).retrieveBytes(FILE_ID);
}
@Test
@DisplayName(
"getFileMetadata: findJobKeyByFileId throws (backplane down) → 503 + Retry-After,"
+ " not 404/500")
void getFileMetadata_findJobKeyThrows_returns503Retryable() throws Exception {
when(taskManager.findJobKeyByFileId(FILE_ID))
.thenThrow(new RuntimeException("Valkey command timeout"));
ResponseEntity<?> response = makeController().getFileMetadata(FILE_ID);
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
assertEquals("1", response.getHeaders().getFirst("Retry-After"));
Map<?, ?> body = (Map<?, ?>) response.getBody();
assertTrue(((String) body.get("message")).toLowerCase().contains("unavailable"));
verify(fileStorage, never()).retrieveBytes(FILE_ID);
}
}
app/core/src/test/java/stirling/software/common/controller/JobControllerTest.java
+6
View File
@@ -19,6 +19,8 @@ import org.springframework.test.util.ReflectionTestUtils;
import jakarta.servlet.http.HttpServletRequest;
import stirling.software.common.cluster.ClusterBackplane;
import stirling.software.common.cluster.JobStore;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.service.FileStorage;
import stirling.software.common.service.JobOwnershipService;
@@ -37,6 +39,10 @@ class JobControllerTest {
@Mock private JobOwnershipService jobOwnershipService;
@Mock private ClusterBackplane clusterBackplane;
@Mock private JobStore jobStore;
private MockHttpSession session;
@InjectMocks private JobController controller;
app/proprietary/build.gradle
+10 -2
View File
@@ -55,8 +55,13 @@ dependencies {
api 'org.springframework.boot:spring-boot-starter-mail'
api 'org.springframework.boot:spring-boot-starter-cache'
api 'com.github.ben-manes.caffeine:caffeine'
implementation 'org.springframework.boot:spring-boot-starter-data-redis'
api 'io.swagger.core.v3:swagger-core-jakarta:2.2.46'
implementation 'com.bucket4j:bucket4j_jdk17-core:8.18.0'
implementation 'com.bucket4j:bucket4j_jdk17-core:8.19.0'
// Lettuce-backed Bucket4j ProxyManager used by ValkeyRateLimitStore for cluster-wide
// token-bucket rate limiting (parity with in-process Bucket4j semantics; no fixed-window
// boundary doubling).
implementation 'com.bucket4j:bucket4j_jdk17-lettuce:8.19.0'
// https://mvnrepository.com/artifact/com.bucket4j/bucket4j_jdk17
implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion"
@@ -77,9 +82,12 @@ dependencies {
implementation "software.amazon.awssdk:s3:$awsSdkVersion"
implementation "software.amazon.awssdk:url-connection-client:$awsSdkVersion"
// Testcontainers: real MinIO/LocalStack (S3) and Valkey for integration tests in CI without
// manually-started instances. Tests skip cleanly when Docker is unavailable.
testImplementation "org.testcontainers:testcontainers:$testcontainersMinioVersion"
testImplementation "org.testcontainers:minio:$testcontainersMinioVersion"
testImplementation "org.testcontainers:junit-jupiter:$testcontainersMinioVersion"
testImplementation "org.testcontainers:localstack:$testcontainersMinioVersion"
testImplementation "org.testcontainers:junit-jupiter:$testcontainersMinioVersion"
}
tasks.register('prepareKotlinBuildScriptModel') {}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterLicenseGate.java
+40
View File
@@ -0,0 +1,40 @@
package stirling.software.proprietary.cluster;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
/**
* Runtime license gate for cluster mode. Cluster mode requires a SERVER or ENTERPRISE license; the
* SaaS flavor bypasses (no {@code runningProOrHigher} bean is published). The Valkey connection
* config {@code @DependsOn} this bean, so it runs before any Valkey bean is constructed.
*/
@Configuration
@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
@Slf4j
public class ClusterLicenseGate {
@Autowired(required = false)
@Qualifier("runningProOrHigher")
private Boolean runningProOrHigher;
@PostConstruct
void verifyLicense() {
if (runningProOrHigher == null) {
return; // saas flavor - licensed via Stripe elsewhere
}
if (!runningProOrHigher) {
throw new IllegalStateException(
"Cluster mode (cluster.enabled=true) requires a SERVER or"
+ " ENTERPRISE license. Configure stirling.premium.key with a valid"
+ " license key (contact [email protected] to obtain one), or set"
+ " cluster.enabled=false.");
}
log.info("Cluster license gate: SERVER/ENTERPRISE license verified, cluster mode allowed.");
}
}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterMetrics.java
+113
View File
@@ -0,0 +1,113 @@
package stirling.software.proprietary.cluster;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import stirling.software.common.cluster.StickyMissRecorder;
import stirling.software.common.model.ApplicationProperties;
/**
* Cluster operation metrics exposed via {@code /actuator/prometheus}. Registered only when cluster
* mode is on.
*/
@Component
@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
public class ClusterMetrics implements StickyMissRecorder {
private final MeterRegistry registry;
private final ApplicationProperties applicationProperties;
private final Counter stickyMissTotal;
private final Counter rateLimitRejected;
private final Timer backplaneLatency;
private final Timer jobWaitSeconds;
// Per-lane queue depth gauges. Lanes are a fixed enum (FAST, SLOW, AI), so we register all
// three eagerly so dashboards never have a missing series.
private static final List<String> KNOWN_LANES = List.of("FAST", "SLOW", "AI");
private final ConcurrentHashMap<String, AtomicLong> queueDepth = new ConcurrentHashMap<>();
private final AtomicLong jobsInflight = new AtomicLong();
public ClusterMetrics(MeterRegistry registry, ApplicationProperties applicationProperties) {
this.registry = registry;
this.applicationProperties = applicationProperties;
this.stickyMissTotal =
Counter.builder("stirling_cluster_sticky_miss_total")
.description(
"Sticky-session misses: a download for a job whose result lives on"
+ " a peer node landed on this node. High sustained value means"
+ " LB affinity is broken.")
.register(registry);
this.rateLimitRejected =
Counter.builder("stirling_cluster_ratelimit_rejected_total")
.description("Cluster-wide rate limit rejections")
.register(registry);
this.backplaneLatency =
Timer.builder("stirling_cluster_backplane_latency_seconds")
.description("Backplane round-trip latency")
.register(registry);
this.jobWaitSeconds =
Timer.builder("stirling_cluster_job_wait_seconds")
.description("Time jobs spend queued before execution")
.register(registry);
Gauge.builder("stirling_cluster_jobs_inflight", jobsInflight, AtomicLong::doubleValue)
.description("Jobs currently in flight on this node")
.tag("node", applicationProperties.getCluster().resolvedNodeId())
.register(registry);
for (String lane : KNOWN_LANES) {
ensureLaneGauge(lane);
}
}
@Override
public void recordStickyMiss() {
stickyMissTotal.increment();
}
public void recordRateLimitReject() {
rateLimitRejected.increment();
}
public Timer backplaneLatency() {
return backplaneLatency;
}
public Timer jobWaitSeconds() {
return jobWaitSeconds;
}
public void incrementInflight() {
jobsInflight.incrementAndGet();
}
public void decrementInflight() {
jobsInflight.decrementAndGet();
}
public void setQueueDepth(String lane, long depth) {
ensureLaneGauge(lane).set(depth);
}
private AtomicLong ensureLaneGauge(String lane) {
return queueDepth.computeIfAbsent(
lane,
l -> {
AtomicLong holder = new AtomicLong();
Gauge.builder("stirling_cluster_queue_depth", holder, AtomicLong::doubleValue)
.description("Pending items in a job queue lane")
.tag("lane", l)
.register(registry);
return holder;
});
}
}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterNodeBootstrap.java
+201
View File
@@ -0,0 +1,201 @@
package stirling.software.proprietary.cluster;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.time.Duration;
import java.time.Instant;
import java.util.Locale;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.SmartLifecycle;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.ClusterNode;
import stirling.software.common.cluster.InstanceRegistry;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.ApplicationProperties.Cluster;
/**
* Registers the local node with {@link InstanceRegistry} on startup, refreshes the entry at 1/3 of
* the TTL, and deregisters cleanly on shutdown.
*
* <p>Implements {@link SmartLifecycle} with {@code getPhase() == Integer.MAX_VALUE} so Spring tears
* this bean down before {@code LettuceConnectionFactory} - deregister therefore runs while the
* Valkey connection is still alive.
*/
@Component
@Slf4j
@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
public class ClusterNodeBootstrap implements SmartLifecycle {
private final Duration heartbeatTtl;
private final ApplicationProperties applicationProperties;
private final InstanceRegistry instanceRegistry;
@Value("${server.port:8080}")
private int serverPort;
private volatile String nodeId;
private volatile String internalAddress;
private volatile boolean running = false;
public ClusterNodeBootstrap(
ApplicationProperties applicationProperties, InstanceRegistry instanceRegistry) {
this.applicationProperties = applicationProperties;
this.instanceRegistry = instanceRegistry;
Cluster cluster = applicationProperties.getCluster();
// Default must match the @Scheduled fallback below AND the model default
// (ApplicationProperties.Cluster.Node.heartbeatIntervalMs = 5000); otherwise the TTL is
// computed from a different interval than the scheduler runs at and the 3x margin breaks.
long heartbeatMs =
cluster.getNode() == null ? 5000L : cluster.getNode().getHeartbeatIntervalMs();
// TTL = 3x heartbeat: tolerate two missed ticks before the node drops out of the registry.
this.heartbeatTtl = Duration.ofMillis(heartbeatMs * 3);
}
@EventListener(ApplicationReadyEvent.class)
public void registerOnStartup() {
nodeId = applicationProperties.getCluster().resolvedNodeId();
internalAddress = resolveInternalAddress();
registerSelf("register");
}
@Scheduled(fixedDelayString = "${cluster.node.heartbeat-interval-ms:5000}")
public void heartbeat() {
// Heartbeat-after-stop race: SmartLifecycle.stop() deregisters, but the @Scheduled
// tick keeps firing during a slow drain. Without this guard, the next tick re-registers
// the dead node and the entry resurfaces in the registry until TTL expiry.
if (!running) {
return;
}
if (nodeId == null) {
return; // not yet registered (startup race)
}
// Self-healing: register() is idempotent and re-populates every field, so a wiped
// Valkey (FLUSHALL, hash eviction) recovers on the next tick without operator action.
registerSelf("heartbeat");
}
private void registerSelf(String reason) {
try {
instanceRegistry.register(
new ClusterNode(nodeId, internalAddress, Instant.now(), role()), heartbeatTtl);
if ("register".equals(reason)) {
log.info(
"Cluster node registered: nodeId={}, internalAddress={}, role={}, ttl={}s",
nodeId,
internalAddress,
role(),
heartbeatTtl.toSeconds());
}
} catch (RuntimeException e) {
log.debug("Cluster {} failed for {}", reason, nodeId, e);
}
}
@Override
public void start() {
running = true;
}
@Override
public void stop() {
running = false;
if (nodeId == null) {
return;
}
try {
instanceRegistry.deregister(nodeId);
log.info("Cluster node deregistered: {}", nodeId);
} catch (RuntimeException e) {
// Registry entry will TTL-expire within heartbeatTtl anyway.
log.warn(
"Cluster deregister failed for {} (will TTL-expire within {}s): {}",
nodeId,
heartbeatTtl.toSeconds(),
e.getMessage());
}
}
@Override
public boolean isRunning() {
return running;
}
@Override
public int getPhase() {
return Integer.MAX_VALUE;
}
@Override
public boolean isAutoStartup() {
return true;
}
/**
* Resolve the address peers should hit. Order: explicit config -> {@code POD_IP} env (K8s
* downward API) -> JDK hostname -> fail loud (never silently fall back to a loopback).
*
* <p>Scheme is taken from {@code cluster.node.scheme} (default {@code http}). Set to {@code
* https} when nodes terminate TLS themselves; leave as {@code http} when an upstream LB
* terminates TLS and intra-cluster traffic is plain HTTP.
*/
private String resolveInternalAddress() {
Cluster cluster = applicationProperties.getCluster();
String configured =
cluster.getNode() == null ? null : cluster.getNode().getInternalAddress();
if (configured != null && !configured.isBlank()) {
return ensurePort(configured);
}
String podIp = System.getenv("POD_IP");
if (podIp != null && !podIp.isBlank()) {
return scheme() + "://" + podIp + ":" + serverPort;
}
try {
return scheme()
+ "://"
+ InetAddress.getLocalHost().getHostAddress()
+ ":"
+ serverPort;
} catch (UnknownHostException e) {
throw new IllegalStateException(
"Could not resolve this host's address for cluster registration; set"
+ " cluster.node.internal-address explicitly (or set POD_IP"
+ " in the Kubernetes downward API).",
e);
}
}
private String ensurePort(String addr) {
if (addr.startsWith("http://") || addr.startsWith("https://")) {
return addr;
}
if (addr.contains(":")) {
return scheme() + "://" + addr;
}
return scheme() + "://" + addr + ":" + serverPort;
}
private String scheme() {
Cluster cluster = applicationProperties.getCluster();
if (cluster.getNode() == null
|| cluster.getNode().getScheme() == null
|| cluster.getNode().getScheme().isBlank()) {
return "http";
}
String s = cluster.getNode().getScheme().trim().toLowerCase(Locale.ROOT);
return "https".equals(s) ? "https" : "http";
}
private String role() {
Cluster.NodeRole r = applicationProperties.getCluster().resolvedRole();
return r == null ? "BOTH" : r.name();
}
}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ConditionalOnValkeyBackplane.java
+19
View File
@@ -0,0 +1,19 @@
package stirling.software.proprietary.cluster.valkey;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
/**
* Composite condition: matches only when cluster.enabled=true AND cluster.backplane=valkey. Both
* checks are required (enabled alone may select the in-process backplane, which must not load
* Valkey beans); a single {@code @ConditionalOnExpression} keeps the guard in one place.
*/
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@ConditionalOnExpression(
"${cluster.enabled:false} and '${cluster.backplane:inprocess}'.equals('valkey')")
public @interface ConditionalOnValkeyBackplane {}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplane.java
+55
View File
@@ -0,0 +1,55 @@
package stirling.software.proprietary.cluster.valkey;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.ClusterBackplane;
import stirling.software.common.model.ApplicationProperties;
@Slf4j
@Component
@RequiredArgsConstructor
@ConditionalOnValkeyBackplane
public class ValkeyClusterBackplane implements ClusterBackplane {
private final ApplicationProperties applicationProperties;
private final StringRedisTemplate template;
@Override
public boolean isHealthy() {
try {
// template.execute() borrows from the pool and returns the connection in a finally
// block - critical because isHealthy() is hit on every k8s liveness/readiness probe
// tick. Calling getConnectionFactory().getConnection() directly leaks the connection
// and exhausts the pool under monitoring load.
String pong = template.execute((RedisCallback<String>) connection -> connection.ping());
return "PONG".equalsIgnoreCase(pong);
} catch (RuntimeException ex) {
log.warn("Valkey backplane health check failed: {}", ex.getMessage());
return false;
}
}
@Override
public String backplaneType() {
return "valkey";
}
@Override
public String localNodeId() {
return applicationProperties.getCluster().resolvedNodeId();
}
/**
* Valkey TTL evicts job entries; local cleanup loop is redundant and would race with cluster
* state.
*/
@Override
public boolean shouldRunLocalCleanup() {
return false;
}
}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfiguration.java
+265
View File
@@ -0,0 +1,265 @@
package stirling.software.proprietary.cluster.valkey;
import java.net.URI;
import java.net.URISyntaxException;
import java.time.Duration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.connection.RedisPassword;
import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.core.StringRedisTemplate;
import io.lettuce.core.RedisCommandExecutionException;
import io.lettuce.core.SslVerifyMode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.ApplicationProperties.Cluster;
@Slf4j
@Configuration
@RequiredArgsConstructor
@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
@DependsOn("clusterLicenseGate")
public class ValkeyConnectionConfiguration {
private final ApplicationProperties applicationProperties;
@Bean(destroyMethod = "destroy")
@ConditionalOnProperty(name = "cluster.backplane", havingValue = "valkey")
public LettuceConnectionFactory valkeyConnectionFactory() {
Cluster cluster = applicationProperties.getCluster();
Endpoint endpoint = parseUrl(cluster.getValkey().getUrl());
RedisStandaloneConfiguration cfg =
new RedisStandaloneConfiguration(endpoint.host(), endpoint.port());
if (endpoint.username() != null) {
cfg.setUsername(endpoint.username());
}
if (endpoint.password() != null) {
cfg.setPassword(RedisPassword.of(endpoint.password()));
}
boolean skipCertVerification =
cluster.getValkey().getTls() != null
&& cluster.getValkey().getTls().isSkipCertVerification();
LettuceClientConfiguration clientConfig =
buildClientConfiguration(endpoint.tls(), skipCertVerification);
LettuceConnectionFactory factory = new LettuceConnectionFactory(cfg, clientConfig);
factory.afterPropertiesSet();
// Eager handshake with retry tolerates docker-compose DNS races; fails boot loudly
// if Valkey is genuinely unreachable.
eagerHandshake(factory, endpoint.host(), endpoint.port(), endpoint.tls());
log.info(
"Valkey connection configured: {}:{} tls={} verifyPeer={}",
endpoint.host(),
endpoint.port(),
endpoint.tls(),
endpoint.tls() ? clientConfig.getVerifyMode() : "n/a");
return factory;
}
/** Parsed connection endpoint; username/password are null when absent. */
record Endpoint(String host, int port, boolean tls, String username, String password) {}
/**
* Parses {@code redis://[user:password@]host[:port]} (or {@code rediss://} for TLS) into an
* {@link Endpoint}. Package-private and side-effect-free so URL handling is unit-testable.
*
* <ul>
* <li>Missing port defaults to 6379.
* <li>{@code rediss} scheme selects TLS.
* <li>Userinfo {@code :password@} (empty user) is treated as password-only auth against the
* default user, not a login with an empty username.
* <li>Reserved characters in the password ({@code @ : / # ?}) must be percent-encoded; {@link
* URI} parses them structurally otherwise (e.g. {@code #} starts the fragment).
* </ul>
*
* @throws IllegalStateException if the URL is blank, syntactically invalid, or has no host
*/
static Endpoint parseUrl(String url) {
if (url == null || url.isBlank()) {
throw new IllegalStateException("cluster.valkey.url must be set when backplane=valkey");
}
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException ex) {
throw new IllegalStateException(
"cluster.valkey.url is not a valid URI: " + url + " (" + ex.getMessage() + ")",
ex);
}
String host = uri.getHost();
if (host == null || host.isBlank()) {
throw new IllegalStateException(
"cluster.valkey.url has no host: "
+ url
+ " (expected redis://[user:password@]host[:port])");
}
boolean tls = "rediss".equalsIgnoreCase(uri.getScheme());
int port = uri.getPort() <= 0 ? 6379 : uri.getPort();
String username = null;
String password = null;
String userInfo = uri.getUserInfo();
if (userInfo != null) {
String[] parts = userInfo.split(":", 2);
if (parts.length == 2) {
username = parts[0].isEmpty() ? null : parts[0];
password = parts[1];
} else if (!parts[0].isBlank()) {
password = parts[0];
}
}
return new Endpoint(host, port, tls, username, password);
}
/**
* Package-private for testing. verifyPeer(FULL) is pinned explicitly so a Spring Data Redis
* default change cannot silently weaken our TLS handshake. skipCertVerification is dev-only.
*/
static LettuceClientConfiguration buildClientConfiguration(
boolean tls, boolean skipCertVerification) {
LettuceClientConfiguration.LettuceClientConfigurationBuilder clientBuilder =
LettuceClientConfiguration.builder();
// Bound every backplane command. Lettuce defaults to 60s; without this a partitioned or
// slow Valkey would stall hot-path calls (e.g. JobController.guardNonOwner -> jobStore.get
// on each request) for up to a minute, exhausting request threads. All backplane ops are
// non-blocking single commands, so a short timeout is safe.
clientBuilder.commandTimeout(Duration.ofSeconds(2));
if (tls) {
clientBuilder
.useSsl()
.verifyPeer(skipCertVerification ? SslVerifyMode.NONE : SslVerifyMode.FULL);
if (skipCertVerification) {
log.warn(
"Valkey TLS hostname/chain verification DISABLED via"
+ " cluster.valkey.tls.skip-cert-verification=true"
+ " - insecure, dev-only");
}
}
return clientBuilder.build();
}
/**
* 10 x 3s = 30s boot-time retry. Auth failures (WRONGPASS/NOAUTH/NOPERM) short-circuit
* immediately; only transport errors get the loop. Package-private for testing.
*/
static void eagerHandshake(
LettuceConnectionFactory factory, String host, int port, boolean tls) {
RuntimeException last = null;
for (int attempt = 1; attempt <= 10; attempt++) {
try {
String pong;
RedisConnection conn = factory.getConnection();
try {
pong = conn.ping();
} finally {
conn.close();
}
if (!"PONG".equalsIgnoreCase(pong)) {
throw new IllegalStateException(
"Valkey PING returned '" + pong + "' (expected PONG)");
}
if (attempt > 1) {
log.info("Valkey reachable after {} attempts", attempt);
}
return;
} catch (RuntimeException ex) {
if (isAuthFailure(ex)) {
factory.destroy();
throw new IllegalStateException(
"Valkey authentication failed for "
+ host
+ ":"
+ port
+ " (tls="
+ tls
+ "): "
+ rootAuthMessage(ex)
+ ". Check cluster.valkey.url credentials"
+ " (user/password and ACL permissions).",
ex);
}
last = ex;
log.warn(
"Valkey PING attempt {}/10 failed ({}:{}, tls={}): {}",
attempt,
host,
port,
tls,
ex.getMessage());
try {
Thread.sleep(3000);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
break;
}
}
}
factory.destroy();
throw new IllegalStateException(
"Valkey unreachable at boot after 10 attempts ("
+ host
+ ":"
+ port
+ ", tls="
+ tls
+ "): "
+ (last == null ? "no detail" : last.getMessage()),
last);
}
/**
* Walks the cause chain for WRONGPASS/NOAUTH/NOPERM replies. Spring Data Redis wraps Lettuce's
* RedisCommandExecutionException in RedisSystemException, so the auth signal may be one level
* down. No typed auth exception exists in spring-data-redis 4.0.5 / Lettuce 6.8.2.
*/
static boolean isAuthFailure(Throwable t) {
for (Throwable cur = t; cur != null; cur = cur.getCause()) {
if (cur instanceof RedisCommandExecutionException && hasAuthPrefix(cur.getMessage())) {
return true;
}
if (hasAuthPrefix(cur.getMessage())) {
return true;
}
if (cur.getCause() == cur) {
break;
}
}
return false;
}
private static boolean hasAuthPrefix(String message) {
if (message == null) {
return false;
}
String upper = message.toUpperCase(java.util.Locale.ROOT).stripLeading();
return upper.startsWith("WRONGPASS")
|| upper.startsWith("NOAUTH")
|| upper.startsWith("NOPERM");
}
private static String rootAuthMessage(Throwable t) {
for (Throwable cur = t; cur != null; cur = cur.getCause()) {
if (cur instanceof RedisCommandExecutionException && cur.getMessage() != null) {
return cur.getMessage();
}
if (cur.getCause() == cur) {
break;
}
}
return t.getMessage();
}
@Bean
@ConditionalOnProperty(name = "cluster.backplane", havingValue = "valkey")
public StringRedisTemplate valkeyTemplate(LettuceConnectionFactory factory) {
return new StringRedisTemplate(factory);
}
}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyDistributedLock.java
+102
View File
@@ -0,0 +1,102 @@
package stirling.software.proprietary.cluster.valkey;
import java.time.Duration;
import java.util.Collections;
import java.util.Optional;
import java.util.UUID;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.script.DefaultRedisScript;
import org.springframework.data.redis.core.script.RedisScript;
import org.springframework.stereotype.Component;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.DistributedLock;
@Component
@RequiredArgsConstructor
@ConditionalOnValkeyBackplane
@Slf4j
public class ValkeyDistributedLock implements DistributedLock {
private static final String PREFIX = "stirling:lock:";
private static final RedisScript<Long> RELEASE_SCRIPT =
new DefaultRedisScript<>(
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end",
Long.class);
private static final RedisScript<Long> RENEW_SCRIPT =
new DefaultRedisScript<>(
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end",
Long.class);
private final StringRedisTemplate template;
@Override
public Optional<LockHandle> tryAcquire(String lockKey, Duration leaseTime) {
String key = PREFIX + lockKey;
String value = UUID.randomUUID().toString();
Boolean ok = template.opsForValue().setIfAbsent(key, value, leaseTime);
if (Boolean.TRUE.equals(ok)) {
return Optional.of(new ValkeyHandle(template, key, value));
}
return Optional.empty();
}
private static final class ValkeyHandle implements LockHandle {
private final StringRedisTemplate template;
private final String key;
private final String value;
private boolean released;
ValkeyHandle(StringRedisTemplate template, String key, String value) {
this.template = template;
this.key = key;
this.value = value;
}
@Override
public synchronized void release() {
if (released) {
return;
}
released = true;
// Swallow + log: LockHandle is AutoCloseable, so release() runs from close() inside
// try-with-resources. An uncaught Valkey error here would mask the body's exception.
// The lease TTL-expires anyway, so a failed explicit release is safe.
try {
template.execute(RELEASE_SCRIPT, Collections.singletonList(key), value);
} catch (RuntimeException ex) {
log.warn(
"Lock release failed for {} (lease will TTL-expire): {}",
key,
ex.getMessage());
}
}
@Override
public synchronized boolean renew(Duration leaseTime) {
if (released) {
return false;
}
try {
Long result =
template.execute(
RENEW_SCRIPT,
Collections.singletonList(key),
value,
Long.toString(leaseTime.toMillis()));
return result != null && result == 1L;
} catch (RuntimeException ex) {
log.warn(
"Lock renew failed for {} (treated as lost lease): {}",
key,
ex.getMessage());
return false;
}
}
}
}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyInstanceRegistry.java
+115
View File
@@ -0,0 +1,115 @@
package stirling.software.proprietary.cluster.valkey;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import lombok.RequiredArgsConstructor;
import stirling.software.common.cluster.ClusterNode;
import stirling.software.common.cluster.InstanceRegistry;
/**
* Valkey-backed {@link InstanceRegistry}. Each node is stored as a hash with a TTL equal to the
* configured heartbeat TTL; the heartbeat re-arms the TTL.
*/
@Component
@RequiredArgsConstructor
@ConditionalOnValkeyBackplane
public class ValkeyInstanceRegistry implements InstanceRegistry {
private static final String PREFIX = "stirling:nodes:";
private final StringRedisTemplate template;
@Override
public void register(ClusterNode node, Duration heartbeatTtl) {
String key = PREFIX + node.nodeId();
long ttlMs = heartbeatTtl.toMillis();
Map<String, String> fields = new LinkedHashMap<>();
fields.put("nodeId", node.nodeId());
fields.put("internalAddress", node.internalAddress());
fields.put("role", node.role());
fields.put("lastHeartbeat", node.lastHeartbeat().toString());
// MULTI/EXEC so the hash fields and the TTL commit together. Without this, a crash
// between HSET and EXPIRE leaves the hash with no TTL: it never expires, masks the
// dead node as alive, and only a subsequent successful register() would re-arm it.
template.execute(
(RedisCallback<Object>)
connection -> {
connection.multi();
byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
Map<byte[], byte[]> hashBytes = new LinkedHashMap<>();
for (Map.Entry<String, String> f : fields.entrySet()) {
hashBytes.put(
f.getKey().getBytes(StandardCharsets.UTF_8),
f.getValue().getBytes(StandardCharsets.UTF_8));
}
connection.hashCommands().hMSet(keyBytes, hashBytes);
connection.keyCommands().pExpire(keyBytes, ttlMs);
connection.exec();
return null;
});
}
@Override
public Optional<ClusterNode> lookup(String nodeId) {
return readNode(PREFIX + nodeId);
}
@Override
public Collection<ClusterNode> activeNodes() {
ScanOptions options = ScanOptions.scanOptions().match(PREFIX + "*").count(256).build();
List<ClusterNode> nodes = new ArrayList<>();
try (Cursor<String> cursor = template.scan(options)) {
while (cursor.hasNext()) {
readNode(cursor.next()).ifPresent(nodes::add);
}
}
return nodes;
}
@Override
public void deregister(String nodeId) {
template.delete(PREFIX + nodeId);
}
private Optional<ClusterNode> readNode(String key) {
Map<Object, Object> entries = template.opsForHash().entries(key);
if (entries == null || entries.isEmpty()) {
return Optional.empty();
}
Object nodeId = entries.get("nodeId");
if (nodeId == null) {
return Optional.empty();
}
Instant heartbeat = Instant.now();
Object hb = entries.get("lastHeartbeat");
if (hb != null) {
try {
heartbeat = Instant.parse(hb.toString());
} catch (RuntimeException ignored) {
// keep default
}
}
return Optional.of(
new ClusterNode(
nodeId.toString(),
String.valueOf(entries.getOrDefault("internalAddress", "")),
heartbeat,
String.valueOf(entries.getOrDefault("role", "BOTH"))));
}
}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyJobStore.java
+297
View File
@@ -0,0 +1,297 @@
package stirling.software.proprietary.cluster.valkey;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.JobStore;
import stirling.software.common.cluster.JobStoreEntry;
/**
* Valkey-backed {@link JobStore}. Each job is one hash; a reverse index maps fileId to jobId.
*
* <p><b>put() atomicity:</b> the hash fields, the per-job TTL, and the reverse-index entries are
* issued inside a single pipelined Redis transaction (MULTI/EXEC). A partial failure cannot leave
* the hash without a TTL or with half the file→job index entries written.
*/
@Component
@RequiredArgsConstructor
@ConditionalOnValkeyBackplane
@Slf4j
public class ValkeyJobStore implements JobStore {
private static final String JOB_PREFIX = "stirling:job:";
private static final String FILE_INDEX_PREFIX = "stirling:file2job:";
private static final ObjectMapper MAPPER = new ObjectMapper();
private static final TypeReference<List<String>> LIST_STRING = new TypeReference<>() {};
private static final TypeReference<Map<String, String>> MAP_STRING = new TypeReference<>() {};
private final StringRedisTemplate template;
@Override
public void put(JobStoreEntry entry, Duration ttl) {
String key = JOB_PREFIX + entry.jobId();
long ttlMs = ttl.toMillis();
Map<String, String> fields = new LinkedHashMap<>();
fields.put("jobId", entry.jobId());
fields.put("state", entry.state().name());
fields.put("owningNodeId", entry.owningNodeId() == null ? "" : entry.owningNodeId());
if (entry.createdAt() != null) {
fields.put("createdAt", entry.createdAt().toString());
}
if (entry.completedAt() != null) {
fields.put("completedAt", entry.completedAt().toString());
}
if (entry.error() != null) {
fields.put("error", entry.error());
}
fields.put("fileIds", writeJson(entry.fileIds() == null ? List.of() : entry.fileIds()));
fields.put(
"resultMeta",
writeJson(entry.resultMeta() == null ? Map.of() : entry.resultMeta()));
// Build pipelined MULTI/EXEC so the hash, its TTL, and every reverse-index entry
// commit atomically.
template.execute(
(RedisCallback<Object>)
connection -> {
connection.multi();
byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
Map<byte[], byte[]> hashBytes = new LinkedHashMap<>();
for (Map.Entry<String, String> f : fields.entrySet()) {
hashBytes.put(
f.getKey().getBytes(StandardCharsets.UTF_8),
f.getValue().getBytes(StandardCharsets.UTF_8));
}
connection.hashCommands().hMSet(keyBytes, hashBytes);
connection.keyCommands().pExpire(keyBytes, ttlMs);
if (entry.fileIds() != null) {
for (String fileId : entry.fileIds()) {
byte[] idxKey =
(FILE_INDEX_PREFIX + fileId)
.getBytes(StandardCharsets.UTF_8);
connection
.stringCommands()
.set(
idxKey,
entry.jobId().getBytes(StandardCharsets.UTF_8));
connection.keyCommands().pExpire(idxKey, ttlMs);
}
}
connection.exec();
return null;
});
}
@Override
public Optional<JobStoreEntry> get(String jobId) {
return readEntry(JOB_PREFIX + jobId);
}
@Override
public void delete(String jobId) {
// WATCH/MULTI/EXEC: read fileIds INSIDE the watched scope so a concurrent put() that
// adds new fileIds between our read and EXEC aborts the transaction. Without this guard,
// an interleaved put() that grows fileIds would leave orphaned reverse-index entries
// pointing at the deleted jobId until their TTL expires. One retry handles the common
// case; further contention falls through to lazy TTL cleanup (acceptable - this is an
// eviction path, not a correctness primitive).
String jobKey = JOB_PREFIX + jobId;
byte[] jobKeyBytes = jobKey.getBytes(StandardCharsets.UTF_8);
for (int attempt = 0; attempt < 2; attempt++) {
Boolean committed =
template.execute(
(RedisCallback<Boolean>)
connection -> {
connection.watch(jobKeyBytes);
// Read the single fileIds field with hGet rather than
// hGetAll + map.get: hGetAll returns a Map<byte[],byte[]>
// whose keys compare by identity, so a fresh
// "fileIds".getBytes() lookup never matches and the reverse
// index would be left orphaned. hGet resolves the field
// server-side.
byte[] fileIdsBytes =
connection
.hashCommands()
.hGet(
jobKeyBytes,
"fileIds"
.getBytes(
StandardCharsets
.UTF_8));
List<byte[]> keysToDelete = new ArrayList<>();
keysToDelete.add(jobKeyBytes);
if (fileIdsBytes != null) {
List<String> fileIds =
readJsonList(
new String(
fileIdsBytes,
StandardCharsets.UTF_8),
jobKey);
for (String fileId : fileIds) {
keysToDelete.add(
(FILE_INDEX_PREFIX + fileId)
.getBytes(StandardCharsets.UTF_8));
}
}
connection.multi();
for (byte[] key : keysToDelete) {
connection.keyCommands().del(key);
}
List<Object> results = connection.exec();
// exec() returns null when WATCH detected a concurrent
// write; spring-data-redis surfaces this as either null
// or empty depending on the driver path.
return results != null && !results.isEmpty();
});
if (Boolean.TRUE.equals(committed)) {
return;
}
}
log.warn(
"JobStore.delete({}) lost two WATCH races to concurrent put(); reverse-index"
+ " entries may linger until TTL expiry",
jobId);
}
@Override
public boolean exists(String jobId) {
Boolean exists = template.hasKey(JOB_PREFIX + jobId);
return Boolean.TRUE.equals(exists);
}
@Override
public Optional<String> findJobIdByFileId(String fileId) {
return Optional.ofNullable(template.opsForValue().get(FILE_INDEX_PREFIX + fileId));
}
@Override
public Collection<JobStoreEntry> all() {
// SCAN, not KEYS - KEYS blocks the Valkey server for the duration of the walk.
ScanOptions options = ScanOptions.scanOptions().match(JOB_PREFIX + "*").count(256).build();
List<JobStoreEntry> result = new ArrayList<>();
try (Cursor<String> cursor = template.scan(options)) {
while (cursor.hasNext()) {
readEntry(cursor.next()).ifPresent(result::add);
}
}
return result;
}
private Optional<JobStoreEntry> readEntry(String key) {
Map<Object, Object> entries = template.opsForHash().entries(key);
if (entries == null || entries.isEmpty()) {
return Optional.empty();
}
Object jobId = entries.get("jobId");
if (jobId == null) {
return Optional.empty();
}
Instant createdAt = parseInstant(entries.get("createdAt"), key, "createdAt");
Instant completedAt = parseInstant(entries.get("completedAt"), key, "completedAt");
List<String> fileIds = parseList(entries.get("fileIds"), key);
Map<String, String> resultMeta = parseMap(entries.get("resultMeta"), key);
String stateName =
String.valueOf(
entries.getOrDefault("state", JobStoreEntry.JobState.PENDING.name()));
JobStoreEntry.JobState state;
try {
state = JobStoreEntry.JobState.valueOf(stateName);
} catch (IllegalArgumentException ex) {
log.warn("Unrecognised job state '{}' in {}, defaulting to PENDING", stateName, key);
state = JobStoreEntry.JobState.PENDING;
}
String owningNodeId = String.valueOf(entries.getOrDefault("owningNodeId", ""));
String error = entries.get("error") == null ? null : entries.get("error").toString();
return Optional.of(
new JobStoreEntry(
jobId.toString(),
state,
owningNodeId,
createdAt,
completedAt,
error,
fileIds,
resultMeta));
}
private Instant parseInstant(Object v, String key, String field) {
if (v == null) {
return null;
}
try {
return Instant.parse(v.toString());
} catch (RuntimeException e) {
log.warn(
"JobStore {} field '{}' has malformed timestamp '{}' - treating as missing",
key,
field,
v);
return null;
}
}
private List<String> parseList(Object v, String key) {
if (v == null) {
return new ArrayList<>();
}
return readJsonList(v.toString(), key);
}
private Map<String, String> parseMap(Object v, String key) {
if (v == null) {
return new HashMap<>();
}
try {
return MAPPER.readValue(v.toString(), MAP_STRING);
} catch (JsonProcessingException e) {
log.warn(
"JobStore {} field 'resultMeta' is not valid JSON '{}' - treating as empty",
key,
v);
return new HashMap<>();
}
}
private static String writeJson(Object value) {
try {
return MAPPER.writeValueAsString(value);
} catch (JsonProcessingException e) {
throw new IllegalStateException("Failed to JSON-serialize JobStore field", e);
}
}
private List<String> readJsonList(String json, String key) {
try {
List<String> parsed = MAPPER.readValue(json, LIST_STRING);
return parsed == null ? new ArrayList<>() : parsed;
} catch (JsonProcessingException e) {
log.warn(
"JobStore {} field 'fileIds' is not valid JSON '{}' - treating as empty",
key,
json);
return new ArrayList<>();
}
}
}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyKeyValueCache.java
+61
View File
@@ -0,0 +1,61 @@
package stirling.software.proprietary.cluster.valkey;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import lombok.RequiredArgsConstructor;
import stirling.software.common.cluster.KeyValueCache;
@Component
@RequiredArgsConstructor
@ConditionalOnValkeyBackplane
public class ValkeyKeyValueCache implements KeyValueCache {
private static final String PREFIX = "stirling:kv:";
private final StringRedisTemplate template;
@Override
public void put(String namespace, String key, String value, Duration ttl) {
template.opsForValue()
.set(buildKey(namespace, key), value, ttl.toMillis(), TimeUnit.MILLISECONDS);
}
@Override
public Optional<String> get(String namespace, String key) {
return Optional.ofNullable(template.opsForValue().get(buildKey(namespace, key)));
}
@Override
public void evict(String namespace, String key) {
template.delete(buildKey(namespace, key));
}
@Override
public void evictNamespace(String namespace) {
ScanOptions options =
ScanOptions.scanOptions().match(PREFIX + namespace + ":*").count(256).build();
List<String> keys = new ArrayList<>();
try (Cursor<String> cursor = template.scan(options)) {
while (cursor.hasNext()) {
keys.add(cursor.next());
}
}
if (!keys.isEmpty()) {
template.delete(keys);
}
}
private String buildKey(String namespace, String key) {
return PREFIX + namespace + ":" + key;
}
}
app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyRateLimitStore.java
+83
View File
@@ -0,0 +1,83 @@
package stirling.software.proprietary.cluster.valkey;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.stereotype.Component;
import io.github.bucket4j.BucketConfiguration;
import io.github.bucket4j.ConsumptionProbe;
import io.github.bucket4j.distributed.BucketProxy;
import io.github.bucket4j.distributed.ExpirationAfterWriteStrategy;
import io.github.bucket4j.distributed.proxy.ProxyManager;
import io.github.bucket4j.redis.lettuce.Bucket4jLettuce;
import io.lettuce.core.AbstractRedisClient;
import io.lettuce.core.RedisClient;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import stirling.software.common.cluster.RateLimitStore;
/**
* Valkey-backed token-bucket rate limiting via Bucket4j's Lettuce ProxyManager. The token bucket
* refills continuously and enforces one global limit across nodes, with the same semantics as the
* in-process {@code InProcessRateLimitStore} (which also uses Bucket4j).
*/
@Component
@ConditionalOnValkeyBackplane
public class ValkeyRateLimitStore implements RateLimitStore {
private static final String PREFIX = "stirling:rl:";
private final LettuceConnectionFactory connectionFactory;
private ProxyManager<byte[]> proxyManager;
public ValkeyRateLimitStore(LettuceConnectionFactory connectionFactory) {
this.connectionFactory = connectionFactory;
}
@PostConstruct
void initProxyManager() {
AbstractRedisClient client = connectionFactory.getNativeClient();
if (!(client instanceof RedisClient redisClient)) {
throw new IllegalStateException(
"ValkeyRateLimitStore requires a standalone Lettuce RedisClient; got "
+ (client == null ? "null" : client.getClass().getName())
+ " (cluster client not supported by this rate limit impl)");
}
// Expire idle bucket keys so they do not accumulate forever in Valkey (one key per
// user / API-key / IP). TTL tracks the time to refill the bucket from empty, capped at
// 25h to cover the longest (daily) rate-limit window; an idle bucket evicts after that.
this.proxyManager =
Bucket4jLettuce.casBasedBuilder(redisClient)
.expirationAfterWrite(
ExpirationAfterWriteStrategy.basedOnTimeForRefillingBucketUpToMax(
Duration.ofHours(25)))
.build();
}
@PreDestroy
void shutdown() {
proxyManager = null;
}
@Override
public RateLimitDecision tryConsume(String bucketKey, long capacity, Duration refillPeriod) {
byte[] key = (PREFIX + bucketKey).getBytes(StandardCharsets.UTF_8);
BucketConfiguration cfg =
BucketConfiguration.builder()
.addLimit(
stage ->
stage.capacity(capacity)
.refillGreedy(capacity, refillPeriod))
.build();
BucketProxy bucket = proxyManager.builder().build(key, () -> cfg);
ConsumptionProbe probe = bucket.tryConsumeAndReturnRemaining(1);
if (probe.isConsumed()) {
return new RateLimitDecision(true, probe.getRemainingTokens(), 0L);
}
return new RateLimitDecision(false, 0L, probe.getNanosToWaitForRefill());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterLicenseGateTest.java
+62
View File
@@ -0,0 +1,62 @@
package stirling.software.proprietary.cluster;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import org.junit.jupiter.api.Test;
class ClusterLicenseGateTest {
private void injectRunningProOrHigher(ClusterLicenseGate gate, Boolean value) throws Exception {
Field f = ClusterLicenseGate.class.getDeclaredField("runningProOrHigher");
f.setAccessible(true);
f.set(gate, value);
}
private void invokeVerify(ClusterLicenseGate gate) throws Throwable {
Method m = ClusterLicenseGate.class.getDeclaredMethod("verifyLicense");
m.setAccessible(true);
try {
m.invoke(gate);
} catch (InvocationTargetException e) {
throw e.getCause();
}
}
@Test
void serverOrEnterpriseLicense_allowsClusterMode() throws Throwable {
ClusterLicenseGate gate = new ClusterLicenseGate();
injectRunningProOrHigher(gate, Boolean.TRUE);
assertDoesNotThrow(() -> invokeVerify(gate));
}
@Test
void normalLicense_refusesClusterMode_withActionableMessage() throws Exception {
ClusterLicenseGate gate = new ClusterLicenseGate();
injectRunningProOrHigher(gate, Boolean.FALSE);
IllegalStateException ex =
assertThrows(IllegalStateException.class, () -> invokeVerify(gate));
String msg = ex.getMessage();
// The error message must tell the operator exactly what to do.
assertTrue(msg.contains("SERVER"), "message must mention SERVER license tier: " + msg);
assertTrue(msg.contains("ENTERPRISE"), "message must mention ENTERPRISE tier: " + msg);
assertTrue(
msg.contains("stirling.premium.key") || msg.contains("license key"),
"message must explain how to set the license: " + msg);
assertTrue(
msg.contains("cluster.enabled=false"),
"message must offer the opt-out (disable cluster): " + msg);
}
@Test
void saasFlavor_bypassesGate_whenRunningProOrHigherBeanAbsent() throws Throwable {
// In saas builds the runningProOrHigher bean is absent (@Autowired required=false -> null).
ClusterLicenseGate gate = new ClusterLicenseGate();
assertDoesNotThrow(() -> invokeVerify(gate));
}
}
app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterMetricsTest.java
+122
View File
@@ -0,0 +1,122 @@
package stirling.software.proprietary.cluster;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import stirling.software.common.model.ApplicationProperties;
/** Verifies every cluster metric is registered and recorder methods write to them. */
class ClusterMetricsTest {
private SimpleMeterRegistry registry;
private ClusterMetrics metrics;
private static final String NODE = "test-node";
@BeforeEach
void setUp() {
registry = new SimpleMeterRegistry();
ApplicationProperties props = new ApplicationProperties();
props.getCluster().getNode().setId(NODE);
metrics = new ClusterMetrics(registry, props);
}
@Test
void registersAllRequiredMeters() {
assertNotNull(registry.find("stirling_cluster_sticky_miss_total").counter());
assertNotNull(registry.find("stirling_cluster_ratelimit_rejected_total").counter());
assertNotNull(registry.find("stirling_cluster_backplane_latency_seconds").timer());
assertNotNull(registry.find("stirling_cluster_job_wait_seconds").timer());
Gauge inflight = registry.find("stirling_cluster_jobs_inflight").tag("node", NODE).gauge();
assertNotNull(inflight, "jobs_inflight gauge with node tag must be registered eagerly");
}
@Test
void registersKnownLaneGaugesEagerly() {
for (String lane : new String[] {"FAST", "SLOW", "AI"}) {
Gauge g = registry.find("stirling_cluster_queue_depth").tag("lane", lane).gauge();
assertNotNull(g, "lane gauge must be eagerly registered for " + lane);
assertEquals(0.0, g.value(), "lane gauge default value must be 0 for " + lane);
}
assertEquals(
3,
registry.find("stirling_cluster_queue_depth").gauges().size(),
"exactly the three known lane gauges should be registered at boot");
}
@Test
void recordStickyMissIncrementsCounter() {
metrics.recordStickyMiss();
metrics.recordStickyMiss();
assertEquals(2.0, registry.find("stirling_cluster_sticky_miss_total").counter().count());
}
@Test
void recordRateLimitRejectIncrementsCounter() {
metrics.recordRateLimitReject();
assertEquals(
1.0, registry.find("stirling_cluster_ratelimit_rejected_total").counter().count());
}
@Test
void incrementAndDecrementInflightUpdatesGauge() {
metrics.incrementInflight();
metrics.incrementInflight();
metrics.incrementInflight();
metrics.decrementInflight();
Gauge gauge = registry.find("stirling_cluster_jobs_inflight").tag("node", NODE).gauge();
assertEquals(2.0, gauge.value(), "expected 2 inflight after 3 inc / 1 dec");
}
@Test
void setQueueDepthUpdatesEagerlyRegisteredLaneGauge() {
metrics.setQueueDepth("FAST", 4);
metrics.setQueueDepth("SLOW", 7);
Gauge fast = registry.find("stirling_cluster_queue_depth").tag("lane", "FAST").gauge();
Gauge slow = registry.find("stirling_cluster_queue_depth").tag("lane", "SLOW").gauge();
assertEquals(4.0, fast.value());
assertEquals(7.0, slow.value());
}
@Test
void setQueueDepthForUnknownLane_lazyRegistersFallbackGauge() {
metrics.setQueueDepth("custom-lane", 5);
Gauge g = registry.find("stirling_cluster_queue_depth").tag("lane", "custom-lane").gauge();
assertNotNull(g);
assertEquals(5.0, g.value());
}
@Test
void setQueueDepthIsIdempotentAcrossCalls() {
metrics.setQueueDepth("FAST", 1);
metrics.setQueueDepth("FAST", 2);
metrics.setQueueDepth("FAST", 9);
assertEquals(
1,
registry.find("stirling_cluster_queue_depth").tag("lane", "FAST").gauges().size());
assertEquals(
9.0,
registry.find("stirling_cluster_queue_depth").tag("lane", "FAST").gauge().value());
}
@Test
void backplaneLatencyTimerAcceptsRecordings() {
metrics.backplaneLatency().record(java.time.Duration.ofMillis(7));
metrics.backplaneLatency().record(java.time.Duration.ofMillis(11));
assertEquals(
2L, registry.find("stirling_cluster_backplane_latency_seconds").timer().count());
}
@Test
void jobWaitTimerAcceptsRecordings() {
metrics.jobWaitSeconds().record(java.time.Duration.ofMillis(50));
assertEquals(1L, registry.find("stirling_cluster_job_wait_seconds").timer().count());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterNodeBootstrapTest.java
+115
View File
@@ -0,0 +1,115 @@
package stirling.software.proprietary.cluster;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.time.Duration;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.test.util.ReflectionTestUtils;
import stirling.software.common.cluster.ClusterNode;
import stirling.software.common.cluster.InstanceRegistry;
import stirling.software.common.model.ApplicationProperties;
/** Verifies the bootstrap registers / heartbeats / deregisters as expected. */
class ClusterNodeBootstrapTest {
private InstanceRegistry registry;
private ApplicationProperties props;
private ClusterNodeBootstrap bootstrap;
@BeforeEach
void setUp() {
registry = mock(InstanceRegistry.class);
props = new ApplicationProperties();
props.getCluster().setEnabled(true);
props.getCluster().getNode().setId("node-test-1");
props.getCluster().getNode().setRole("worker");
props.getCluster().getNode().setHeartbeatIntervalMs(10_000L);
bootstrap = new ClusterNodeBootstrap(props, registry);
ReflectionTestUtils.setField(bootstrap, "serverPort", 8080);
}
@Test
void registerOnStartupCallsRegistryWithResolvedNodeId() {
bootstrap.registerOnStartup();
ArgumentCaptor<ClusterNode> nodeCaptor = ArgumentCaptor.forClass(ClusterNode.class);
ArgumentCaptor<Duration> ttlCaptor = ArgumentCaptor.forClass(Duration.class);
verify(registry, times(1)).register(nodeCaptor.capture(), ttlCaptor.capture());
ClusterNode captured = nodeCaptor.getValue();
assertEquals("node-test-1", captured.nodeId());
assertTrue(captured.internalAddress().startsWith("http://"));
assertTrue(captured.internalAddress().endsWith(":8080"));
assertEquals("WORKER", captured.role());
assertEquals(30L, ttlCaptor.getValue().toSeconds());
}
@Test
void registerHonoursExplicitInternalAddress() {
props.getCluster().getNode().setInternalAddress("app-1:8080");
bootstrap.registerOnStartup();
ArgumentCaptor<ClusterNode> nodeCaptor = ArgumentCaptor.forClass(ClusterNode.class);
verify(registry).register(nodeCaptor.capture(), any());
assertEquals("http://app-1:8080", nodeCaptor.getValue().internalAddress());
}
@Test
void registerUsesHttpsSchemeWhenConfigured() {
props.getCluster().getNode().setInternalAddress("app-1:8443");
props.getCluster().getNode().setScheme("https");
ClusterNodeBootstrap httpsBootstrap = new ClusterNodeBootstrap(props, registry);
ReflectionTestUtils.setField(httpsBootstrap, "serverPort", 8443);
httpsBootstrap.registerOnStartup();
ArgumentCaptor<ClusterNode> nodeCaptor = ArgumentCaptor.forClass(ClusterNode.class);
verify(registry).register(nodeCaptor.capture(), any());
assertEquals("https://app-1:8443", nodeCaptor.getValue().internalAddress());
}
@Test
void heartbeatAfterStartup_callsRegister_forSelfHealing() {
bootstrap.start();
bootstrap.registerOnStartup();
bootstrap.heartbeat();
verify(registry, times(2))
.register(
any(ClusterNode.class),
org.mockito.ArgumentMatchers.eq(Duration.ofSeconds(30)));
}
@Test
void smartLifecycleStop_deregisters() {
bootstrap.start();
bootstrap.registerOnStartup();
bootstrap.stop();
verify(registry, times(1)).deregister("node-test-1");
}
@Test
void smartLifecycleStop_beforeStartup_isNoop() {
bootstrap.stop();
verify(registry, never()).deregister(any());
}
@Test
void heartbeatAfterStop_doesNotReRegister() {
// Heartbeat-after-stop race: the @Scheduled tick fires during a slow drain. Without
// a guard it would re-register the dead node until TTL expiry.
bootstrap.start();
bootstrap.registerOnStartup();
verify(registry, times(1)).register(any(ClusterNode.class), any(Duration.class));
bootstrap.stop();
verify(registry, times(1)).deregister("node-test-1");
bootstrap.heartbeat();
verify(registry, times(1)).register(any(ClusterNode.class), any(Duration.class));
}
}
app/proprietary/src/test/java/stirling/software/proprietary/cluster/MultiNodeClusterScenarioTest.java
+133
View File
@@ -0,0 +1,133 @@
package stirling.software.proprietary.cluster;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import stirling.software.common.cluster.ClusterBackplane;
import stirling.software.common.cluster.JobStore;
import stirling.software.common.cluster.JobStoreEntry;
import stirling.software.common.cluster.KeyValueCache;
import stirling.software.common.cluster.RateLimitStore;
import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
import stirling.software.common.cluster.inprocess.InProcessJobStore;
import stirling.software.common.cluster.inprocess.InProcessKeyValueCache;
import stirling.software.common.cluster.inprocess.InProcessRateLimitStore;
/**
* Multi-node contract test using in-process impls shared across two "nodes". Verifies cross-node
* visibility, global rate-limit counters, and cache propagation without requiring Docker.
* Valkey-specific behavior (MULTI/EXEC atomicity, WATCH races, TTL) is in
* LiveValkeyIntegrationTest.
*/
class MultiNodeClusterScenarioTest {
private JobStore sharedJobStore;
private RateLimitStore sharedRateLimit;
private KeyValueCache sharedCache;
private ClusterBackplane backplaneA;
private ClusterBackplane backplaneB;
@BeforeEach
void setUp() {
sharedJobStore = new InProcessJobStore();
sharedRateLimit = new InProcessRateLimitStore();
sharedCache = new InProcessKeyValueCache();
backplaneA = constBackplane("node-A", "valkey");
backplaneB = constBackplane("node-B", "valkey");
}
@Test
@DisplayName("async job created on node-A is readable from node-B via shared JobStore")
void jobStatusVisibleCrossNode() {
JobStoreEntry entry =
new JobStoreEntry(
"job-1",
JobStoreEntry.JobState.RUNNING,
"node-A",
Instant.now(),
null,
null,
List.of("file-1"),
Map.of());
sharedJobStore.put(entry, Duration.ofMinutes(30));
Optional<JobStoreEntry> seenOnB = sharedJobStore.get("job-1");
assertTrue(seenOnB.isPresent(), "node-B must see node-A's job in shared JobStore");
assertEquals("node-A", seenOnB.get().owningNodeId());
assertEquals(JobStoreEntry.JobState.RUNNING, seenOnB.get().state());
}
@Test
@DisplayName("global rate limit - capacity counted once across both nodes")
void rateLimitGlobalAcrossNodes() {
long capacity = 4L;
RateLimitDecision a1 =
sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1));
RateLimitDecision b1 =
sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1));
RateLimitDecision a2 =
sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1));
RateLimitDecision b2 =
sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1));
RateLimitDecision a3 =
sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1));
assertTrue(a1.allowed());
assertTrue(b1.allowed());
assertTrue(a2.allowed());
assertTrue(b2.allowed());
assertFalse(a3.allowed(), "5th request across both nodes must be rejected (limit=4)");
}
@Test
@DisplayName("KeyValueCache populated on A is observed on B; evict on A propagates")
void apiKeyCacheVisibleCrossNode() {
sharedCache.put("apikey", "hash-bob", "bob", Duration.ofSeconds(60));
assertEquals("bob", sharedCache.get("apikey", "hash-bob").orElse(null));
sharedCache.evict("apikey", "hash-bob");
assertFalse(sharedCache.get("apikey", "hash-bob").isPresent());
}
@Test
@DisplayName("backplaneType reports 'valkey' on every node; localNodeId is distinct")
void backplaneType() {
assertEquals("valkey", backplaneA.backplaneType());
assertEquals("valkey", backplaneB.backplaneType());
assertEquals("node-A", backplaneA.localNodeId());
assertEquals("node-B", backplaneB.localNodeId());
assertNotEquals(backplaneA.localNodeId(), backplaneB.localNodeId());
assertNotNull(backplaneA.localNodeId());
}
private ClusterBackplane constBackplane(String nodeId, String type) {
return new ClusterBackplane() {
@Override
public boolean isHealthy() {
return true;
}
@Override
public String backplaneType() {
return type;
}
@Override
public String localNodeId() {
return nodeId;
}
};
}
}
app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveExternalClusterTest.java
+237
View File
@@ -0,0 +1,237 @@
package stirling.software.proprietary.cluster.valkey;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.core.StringRedisTemplate;
import stirling.software.common.cluster.ClusterNode;
import stirling.software.common.cluster.DistributedLock;
import stirling.software.common.cluster.JobStoreEntry;
import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
import stirling.software.common.model.ApplicationProperties;
/**
* Opt-in live cluster test against an EXTERNAL Valkey/Redis given by {@code
* STIRLING_TEST_VALKEY_URL} (e.g. a managed {@code rediss://} endpoint). Unlike {@link
* LiveValkeyIntegrationTest} (no-auth local container) this drives three independent node stacks
* through the production {@link ValkeyConnectionConfiguration#valkeyConnectionFactory()} bean - so
* a {@code rediss://} URL exercises the real TLS handshake (verifyPeer=FULL) and credential path
* end to end.
*
* <p>Skips unless the env var is set, so it never runs in normal CI. No secrets are committed.
*/
@EnabledIfEnvironmentVariable(named = "STIRLING_TEST_VALKEY_URL", matches = "rediss?://.+")
class LiveExternalClusterTest {
private static final int NODES = 3;
private static final String RUN = UUID.randomUUID().toString().substring(0, 8);
private static final List<LettuceConnectionFactory> factories = new ArrayList<>();
private static final List<StringRedisTemplate> templates = new ArrayList<>();
@BeforeAll
static void connectAllNodes() {
String url = System.getenv("STIRLING_TEST_VALKEY_URL");
for (int i = 0; i < NODES; i++) {
ApplicationProperties p = new ApplicationProperties();
p.getCluster().setEnabled(true);
p.getCluster().setBackplane("valkey");
p.getCluster().getValkey().setUrl(url);
p.getCluster().getNode().setId(nodeId(i));
// Production bean: parse + credentials + TLS + eager PING handshake (proves reachable).
LettuceConnectionFactory f =
new ValkeyConnectionConfiguration(p).valkeyConnectionFactory();
factories.add(f);
templates.add(new StringRedisTemplate(f));
}
}
@AfterAll
static void disconnect() {
for (LettuceConnectionFactory f : factories) {
try {
f.destroy();
} catch (RuntimeException ignored) {
// best effort
}
}
}
private static String nodeId(int i) {
return "ext-" + RUN + "-node-" + (i + 1);
}
private ApplicationProperties propsForNode(int i) {
ApplicationProperties p = new ApplicationProperties();
p.getCluster().setEnabled(true);
p.getCluster().setBackplane("valkey");
p.getCluster().getNode().setId(nodeId(i));
return p;
}
@Test
@DisplayName("TLS reachability: every node's backplane reports healthy over the external URL")
void allNodesHealthyOverTls() {
for (int i = 0; i < NODES; i++) {
ValkeyClusterBackplane bp =
new ValkeyClusterBackplane(propsForNode(i), templates.get(i));
assertTrue(bp.isHealthy(), nodeId(i) + " must reach the external Valkey (PING)");
assertEquals(nodeId(i), bp.localNodeId());
}
}
@Test
@DisplayName("3-node registry: each node sees all peers; deregister drops one cluster-wide")
void threeNodeRegistryConverges() {
List<ValkeyInstanceRegistry> regs = new ArrayList<>();
for (int i = 0; i < NODES; i++) {
regs.add(new ValkeyInstanceRegistry(templates.get(i)));
}
for (int i = 0; i < NODES; i++) {
regs.get(i)
.register(
new ClusterNode(
nodeId(i), "10.0.0." + i + ":8080", Instant.now(), "BOTH"),
Duration.ofSeconds(30));
}
try {
// Node 0's view must include all three registrations made on three connections.
for (int i = 0; i < NODES; i++) {
final String id = nodeId(i);
boolean seenByNode0 =
regs.get(0).activeNodes().stream().anyMatch(n -> id.equals(n.nodeId()));
assertTrue(seenByNode0, "node-0 must see " + id + " registered by another node");
}
regs.get(1).deregister(nodeId(2));
assertFalse(
regs.get(0).lookup(nodeId(2)).isPresent(),
"deregister on node-1 must be visible from node-0");
} finally {
regs.get(0).deregister(nodeId(0));
regs.get(1).deregister(nodeId(1));
}
}
@Test
@DisplayName(
"Cross-node JobStore: put on node-0 visible on node-1/2; delete on node-2 clears it")
void jobStoreVisibleAcrossNodes() {
ValkeyJobStore s0 = new ValkeyJobStore(templates.get(0));
ValkeyJobStore s1 = new ValkeyJobStore(templates.get(1));
ValkeyJobStore s2 = new ValkeyJobStore(templates.get(2));
String jobId = "ext-job-" + RUN;
String fileId = "ext-file-" + RUN;
s0.put(
new JobStoreEntry(
jobId,
JobStoreEntry.JobState.RUNNING,
nodeId(0),
Instant.now(),
null,
null,
List.of(fileId),
Map.of("k", "v")),
Duration.ofSeconds(60));
Optional<JobStoreEntry> onNode1 = s1.get(jobId);
assertTrue(onNode1.isPresent(), "node-1 must see node-0's job write");
assertEquals(nodeId(0), onNode1.get().owningNodeId());
assertEquals(jobId, s2.findJobIdByFileId(fileId).orElse(null), "reverse index visible too");
s2.delete(jobId);
assertFalse(s0.exists(jobId), "delete on node-2 must clear the hash for node-0");
assertFalse(
s1.findJobIdByFileId(fileId).isPresent(),
"delete on node-2 must clear the reverse index for node-1");
}
@Test
@DisplayName("Global rate limit: ONE shared budget enforced across all three nodes")
void rateLimitGlobalAcrossThreeNodes() {
List<ValkeyRateLimitStore> stores = new ArrayList<>();
for (int i = 0; i < NODES; i++) {
ValkeyRateLimitStore s = new ValkeyRateLimitStore(factories.get(i));
s.initProxyManager();
stores.add(s);
}
String key = "ext-rl-" + RUN;
long capacity = 6;
AtomicInteger allowed = new AtomicInteger();
for (int i = 0; i < 12; i++) {
RateLimitDecision d =
stores.get(i % NODES).tryConsume(key, capacity, Duration.ofSeconds(60));
if (d.allowed()) {
allowed.incrementAndGet();
}
}
assertEquals(
capacity,
allowed.get(),
"exactly the global capacity must be allowed across all three nodes");
}
@Test
@DisplayName("Distributed lock: held by node-0 excludes node-1/2; stale release cannot steal")
void distributedLockAcrossNodes() throws InterruptedException {
ValkeyDistributedLock l0 = new ValkeyDistributedLock(templates.get(0));
ValkeyDistributedLock l1 = new ValkeyDistributedLock(templates.get(1));
ValkeyDistributedLock l2 = new ValkeyDistributedLock(templates.get(2));
String key = "ext-lock-" + RUN;
Optional<DistributedLock.LockHandle> a = l0.tryAcquire(key, Duration.ofSeconds(30));
assertTrue(a.isPresent());
assertFalse(l1.tryAcquire(key, Duration.ofSeconds(30)).isPresent(), "node-1 excluded");
assertFalse(l2.tryAcquire(key, Duration.ofSeconds(30)).isPresent(), "node-2 excluded");
a.get().release();
Optional<DistributedLock.LockHandle> b = l1.tryAcquire(key, Duration.ofSeconds(30));
assertTrue(b.isPresent(), "node-1 acquires after node-0 releases");
// Short lease that expires, then node-2 takes it; node-1's stale release must not steal.
String key2 = "ext-lock2-" + RUN;
Optional<DistributedLock.LockHandle> shortHeld =
l1.tryAcquire(key2, Duration.ofMillis(500));
assertTrue(shortHeld.isPresent());
Thread.sleep(900);
Optional<DistributedLock.LockHandle> stolen = l2.tryAcquire(key2, Duration.ofSeconds(30));
assertTrue(stolen.isPresent(), "node-2 acquires after node-1's lease expires");
shortHeld.get().release(); // value-checked no-op
assertFalse(
l0.tryAcquire(key2, Duration.ofMillis(200)).isPresent(),
"node-2 still holds; node-1's stale release must not have stolen it");
b.get().release();
stolen.get().release();
}
@Test
@DisplayName("KeyValueCache: put on node-0 readable on node-1; evict visible on node-2")
void keyValueCacheAcrossNodes() {
ValkeyKeyValueCache c0 = new ValkeyKeyValueCache(templates.get(0));
ValkeyKeyValueCache c1 = new ValkeyKeyValueCache(templates.get(1));
ValkeyKeyValueCache c2 = new ValkeyKeyValueCache(templates.get(2));
String field = "ext-cache-" + RUN;
c0.put("apikey", field, "alice", Duration.ofSeconds(60));
assertEquals("alice", c1.get("apikey", field).orElse(null), "node-1 reads node-0's cache");
c2.evict("apikey", field);
assertFalse(c0.get("apikey", field).isPresent(), "evict on node-2 visible on node-0");
}
}
app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyAuthIntegrationTest.java
+143
View File
@@ -0,0 +1,143 @@
package stirling.software.proprietary.cluster.valkey;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIf;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import stirling.software.common.model.ApplicationProperties;
/**
* Live AUTH coverage against a password-protected Valkey. The main {@link
* LiveValkeyIntegrationTest} runs against a no-auth instance, so the credential-bearing URL path
* (parse -> RedisStandaloneConfiguration -> real AUTH handshake) was otherwise unexercised. Drives
* the full production bean method {@code valkeyConnectionFactory()} so the parse, credential
* wiring, and eager-handshake all run exactly as at boot.
*/
@Testcontainers
@EnabledIf("isDockerAvailable")
class LiveValkeyAuthIntegrationTest {
private static final String PASSWORD = "s3cr3tpw";
// @Container is intentionally NOT used: lifecycle is managed manually so the container can be
// shared across the password-only and ACL-user cases without a per-method restart.
static final GenericContainer<?> VALKEY =
new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine"))
.withExposedPorts(6379)
.withCommand("valkey-server", "--requirepass", PASSWORD);
static boolean isDockerAvailable() {
return DockerClientFactory.instance().isDockerAvailable();
}
@org.junit.jupiter.api.BeforeAll
static void start() {
VALKEY.start();
}
@org.junit.jupiter.api.AfterAll
static void stop() {
VALKEY.stop();
}
private static String host() {
return VALKEY.getHost();
}
private static int port() {
return VALKEY.getMappedPort(6379);
}
private ApplicationProperties propsWithUrl(String url) {
ApplicationProperties p = new ApplicationProperties();
p.getCluster().setEnabled(true);
p.getCluster().setBackplane("valkey");
p.getCluster().getValkey().setUrl(url);
p.getCluster().getNode().setId("auth-test");
return p;
}
@Test
@DisplayName("password-only URL (redis://:pw@host) authenticates and round-trips a key")
void passwordOnlyAuthWorks() {
String url = "redis://:" + PASSWORD + "@" + host() + ":" + port();
LettuceConnectionFactory factory =
new ValkeyConnectionConfiguration(propsWithUrl(url)).valkeyConnectionFactory();
try {
StringRedisTemplate t = new StringRedisTemplate(factory);
t.opsForValue().set("auth:pwonly", "v1");
assertEquals(
"v1",
t.opsForValue().get("auth:pwonly"),
"password-only AUTH must succeed and the key must round-trip");
} finally {
factory.destroy();
}
}
@Test
@DisplayName("user:password URL (ACL named user) authenticates and round-trips a key")
void namedUserAuthWorks() throws Exception {
// Default user is password-protected; create a named ACL user to exercise two-arg AUTH.
Container.ExecResult res =
VALKEY.execInContainer(
"valkey-cli",
"-a",
PASSWORD,
"ACL",
"SETUSER",
"alice",
"on",
">alicepass",
"~*",
"+@all");
assertEquals(0, res.getExitCode(), "ACL SETUSER failed: " + res.getStderr());
String url = "redis://alice:alicepass@" + host() + ":" + port();
LettuceConnectionFactory factory =
new ValkeyConnectionConfiguration(propsWithUrl(url)).valkeyConnectionFactory();
try {
StringRedisTemplate t = new StringRedisTemplate(factory);
t.opsForValue().set("auth:named", "v2");
assertEquals(
"v2",
t.opsForValue().get("auth:named"),
"named-user AUTH must succeed and the key must round-trip");
} finally {
factory.destroy();
}
}
@Test
@DisplayName(
"wrong password → fast IllegalStateException (real WRONGPASS short-circuits retries)")
void wrongPasswordFailsFast() {
String url = "redis://:wrong-" + PASSWORD + "@" + host() + ":" + port();
ValkeyConnectionConfiguration cfg = new ValkeyConnectionConfiguration(propsWithUrl(url));
long start = System.nanoTime();
IllegalStateException ex =
assertThrows(IllegalStateException.class, cfg::valkeyConnectionFactory);
long elapsedMs = (System.nanoTime() - start) / 1_000_000;
assertTrue(
ex.getMessage().toLowerCase().contains("authentication failed"),
"real Valkey WRONGPASS must be classified as an auth failure; got: "
+ ex.getMessage());
// The retry loop is 10 x 3s; a recognised auth failure must abort well before that.
assertTrue(
elapsedMs < 10_000,
"auth failure must short-circuit the 30s retry loop; elapsed=" + elapsedMs + " ms");
}
}
app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyChaosTest.java
+118
View File
@@ -0,0 +1,118 @@
package stirling.software.proprietary.cluster.valkey;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIf;
import org.springframework.dao.DataAccessException;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import stirling.software.common.model.ApplicationProperties;
/**
* Live failure-injection: a frozen (network-black-holed) Valkey must NOT stall hot-path commands
* for Lettuce's 60s default. {@link ValkeyConnectionConfiguration} pins a 2s command timeout, so a
* paused server must surface an error in seconds, and the connection must recover when it returns.
*
* <p>Uses {@code docker pause}/{@code unpause} (TCP stays ESTABLISHED but the server never replies)
* to reproduce a partition rather than {@code stop} (which would fail fast with
* connection-refused).
*/
@Testcontainers
@EnabledIf("isDockerAvailable")
class LiveValkeyChaosTest {
@Container
static final GenericContainer<?> VALKEY =
new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine"))
.withExposedPorts(6379);
static boolean isDockerAvailable() {
return DockerClientFactory.instance().isDockerAvailable();
}
private boolean paused;
private LettuceConnectionFactory productionFactory() {
ApplicationProperties p = new ApplicationProperties();
p.getCluster().setEnabled(true);
p.getCluster().setBackplane("valkey");
p.getCluster()
.getValkey()
.setUrl("redis://" + VALKEY.getHost() + ":" + VALKEY.getMappedPort(6379));
p.getCluster().getNode().setId("chaos");
// Built via the production bean so the real 2s commandTimeout is in effect.
return new ValkeyConnectionConfiguration(p).valkeyConnectionFactory();
}
private void pause() {
DockerClientFactory.lazyClient().pauseContainerCmd(VALKEY.getContainerId()).exec();
paused = true;
}
private void unpause() {
DockerClientFactory.lazyClient().unpauseContainerCmd(VALKEY.getContainerId()).exec();
paused = false;
}
@AfterEach
void ensureUnpaused() {
if (paused) {
try {
unpause();
} catch (RuntimeException ignored) {
// container teardown will handle it
}
}
}
@Test
@DisplayName("frozen Valkey fails a command in seconds (2s timeout), not Lettuce's 60s default")
void commandTimeoutFiresUnderPartition() {
LettuceConnectionFactory factory = productionFactory();
try {
StringRedisTemplate t = new StringRedisTemplate(factory);
t.opsForValue().set("chaos:k", "before");
assertEquals("before", t.opsForValue().get("chaos:k"));
pause();
long start = System.nanoTime();
// DataAccessException is spring-data-redis's wrapper for the Lettuce timeout.
assertThrows(DataAccessException.class, () -> t.opsForValue().get("chaos:k"));
long elapsedMs = (System.nanoTime() - start) / 1_000_000;
assertTrue(
elapsedMs < 15_000,
"command must abort on the ~2s timeout, not hang on the 60s default; elapsed="
+ elapsedMs
+ " ms");
unpause();
// After the partition heals the client must recover (Lettuce reconnects lazily).
String recovered = null;
long deadline = System.currentTimeMillis() + 10_000;
while (System.currentTimeMillis() < deadline) {
try {
recovered = t.opsForValue().get("chaos:k");
break;
} catch (RuntimeException retry) {
Thread.sleep(250);
}
}
assertEquals("before", recovered, "connection must recover after the partition heals");
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
} finally {
factory.destroy();
}
}
}
app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyIntegrationTest.java
+519
View File
@@ -0,0 +1,519 @@
package stirling.software.proprietary.cluster.valkey;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIf;
import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import stirling.software.common.cluster.ClusterNode;
import stirling.software.common.cluster.DistributedLock;
import stirling.software.common.cluster.JobStoreEntry;
import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
import stirling.software.common.model.ApplicationProperties;
/**
* Live integration tests against a real Valkey instance, started by Testcontainers. The
* {@code @EnabledIf} guard probes the Docker daemon via {@link
* DockerClientFactory#isDockerAvailable()} (non-throwing) so the suite skips cleanly when Docker is
* unavailable - without that guard, {@code @Testcontainers} would throw {@code initializationError}
* (test FAILURE, not skip) on CI runners without Docker.
*/
@Testcontainers
@EnabledIf("isDockerAvailable")
class LiveValkeyIntegrationTest {
@Container
static final GenericContainer<?> VALKEY =
new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine"))
.withExposedPorts(6379);
static boolean isDockerAvailable() {
return DockerClientFactory.instance().isDockerAvailable();
}
private static LettuceConnectionFactory factoryA;
private static LettuceConnectionFactory factoryB;
private static StringRedisTemplate templateA;
private static StringRedisTemplate templateB;
@BeforeAll
static void connect() {
String host = VALKEY.getHost();
int port = VALKEY.getMappedPort(6379);
factoryA = new LettuceConnectionFactory(new RedisStandaloneConfiguration(host, port));
factoryA.afterPropertiesSet();
factoryB = new LettuceConnectionFactory(new RedisStandaloneConfiguration(host, port));
factoryB.afterPropertiesSet();
templateA = new StringRedisTemplate(factoryA);
templateB = new StringRedisTemplate(factoryB);
templateA.getConnectionFactory().getConnection().serverCommands().flushAll();
}
@AfterAll
static void disconnect() {
if (factoryA != null) factoryA.destroy();
if (factoryB != null) factoryB.destroy();
}
@Test
@DisplayName("Valkey reachable and isHealthy() = true after PING round-trip")
void backplaneHealthy() {
ApplicationProperties propsA = newProps("node-A");
ValkeyClusterBackplane bp = new ValkeyClusterBackplane(propsA, templateA);
assertEquals("valkey", bp.backplaneType());
assertEquals("node-A", bp.localNodeId());
assertTrue(bp.isHealthy(), "Valkey must be reachable in the Testcontainers instance");
}
@Test
@DisplayName("JobStore put on connection A, get on connection B reads the same entry")
void jobStoreCrossConnectionVisibility() {
ValkeyJobStore storeA = new ValkeyJobStore(templateA);
ValkeyJobStore storeB = new ValkeyJobStore(templateB);
JobStoreEntry entry =
new JobStoreEntry(
"live-job-1",
JobStoreEntry.JobState.RUNNING,
"node-A",
Instant.now(),
null,
null,
List.of("live-file-1"),
Map.of("k", "v"));
storeA.put(entry, Duration.ofSeconds(30));
Optional<JobStoreEntry> seen = storeB.get("live-job-1");
assertTrue(seen.isPresent(), "storeB on different connection must see storeA's write");
assertEquals("node-A", seen.get().owningNodeId());
assertEquals(JobStoreEntry.JobState.RUNNING, seen.get().state());
assertEquals("live-job-1", storeB.findJobIdByFileId("live-file-1").orElse(null));
}
@Test
@DisplayName("JobStore entry expires after the configured duration")
void jobStoreTtlExpires() throws InterruptedException {
ValkeyJobStore store = new ValkeyJobStore(templateA);
store.put(
new JobStoreEntry(
"ttl-job",
JobStoreEntry.JobState.PENDING,
"node-A",
Instant.now(),
null,
null,
List.of(),
Map.of()),
Duration.ofSeconds(2));
assertTrue(store.exists("ttl-job"));
long deadline = System.currentTimeMillis() + 3000;
boolean expired = false;
while (System.currentTimeMillis() < deadline) {
if (!store.exists("ttl-job")) {
expired = true;
break;
}
Thread.sleep(100);
}
assertTrue(expired, "entry should TTL-expire within 3 s of a 2 s TTL");
}
@Test
@DisplayName("KeyValueCache propagates across connections; evict observed cross-connection")
void keyValueCacheCrossConnection() {
ValkeyKeyValueCache cacheA = new ValkeyKeyValueCache(templateA);
ValkeyKeyValueCache cacheB = new ValkeyKeyValueCache(templateB);
cacheA.put("apikey", "hash-bob", "bob", Duration.ofSeconds(30));
assertEquals("bob", cacheB.get("apikey", "hash-bob").orElse(null));
cacheA.evict("apikey", "hash-bob");
assertFalse(cacheB.get("apikey", "hash-bob").isPresent());
}
@Test
@DisplayName("RateLimitStore enforces ONE global budget across two instances")
void rateLimitGlobalAcrossInstances() {
ValkeyRateLimitStore storeA = newRateLimitStore(factoryA);
ValkeyRateLimitStore storeB = newRateLimitStore(factoryB);
String key = "live-user:alice";
long capacity = 4;
AtomicInteger allowed = new AtomicInteger();
for (int i = 0; i < 8; i++) {
// alternate consumers
var store = (i % 2 == 0) ? storeA : storeB;
RateLimitDecision d = store.tryConsume(key, capacity, Duration.ofSeconds(30));
if (d.allowed()) allowed.incrementAndGet();
}
assertEquals(
4,
allowed.get(),
"exactly 4 (the global capacity) must be allowed across both instances");
}
@Test
@DisplayName("DistributedLock excludes a second acquirer on a different connection")
void distributedLockMutualExclusion() {
ValkeyDistributedLock lockA = new ValkeyDistributedLock(templateA);
ValkeyDistributedLock lockB = new ValkeyDistributedLock(templateB);
Optional<DistributedLock.LockHandle> heldByA =
lockA.tryAcquire("election-X", Duration.ofSeconds(30));
assertTrue(heldByA.isPresent());
Optional<DistributedLock.LockHandle> heldByB =
lockB.tryAcquire("election-X", Duration.ofSeconds(30));
assertFalse(heldByB.isPresent(), "second acquirer must fail while A holds the lock");
heldByA.get().release();
// After release, B can acquire
Optional<DistributedLock.LockHandle> retry =
lockB.tryAcquire("election-X", Duration.ofSeconds(30));
assertTrue(retry.isPresent());
retry.get().release();
}
@Test
@DisplayName("renew() extends the lease TTL well beyond the original")
void distributedLockRenewExtendsLease() {
ValkeyDistributedLock lock = new ValkeyDistributedLock(templateA);
String key = "renew-" + java.util.UUID.randomUUID();
Optional<DistributedLock.LockHandle> held = lock.tryAcquire(key, Duration.ofSeconds(1));
assertTrue(held.isPresent());
assertTrue(held.get().renew(Duration.ofSeconds(30)), "renew on a held lock must succeed");
Long ttlMs =
templateA.getExpire(
"stirling:lock:" + key, java.util.concurrent.TimeUnit.MILLISECONDS);
assertNotNull(ttlMs);
assertTrue(
ttlMs > 1500 && ttlMs <= 30_000,
"renew must reset TTL to the new 30s lease, got " + ttlMs + " ms");
held.get().release();
}
@Test
@DisplayName("value-check prevents a stale owner from releasing/renewing a re-acquired lock")
void distributedLockStealPrevention() throws InterruptedException {
ValkeyDistributedLock lockA = new ValkeyDistributedLock(templateA);
ValkeyDistributedLock lockB = new ValkeyDistributedLock(templateB);
String key = "steal-" + java.util.UUID.randomUUID();
Optional<DistributedLock.LockHandle> a = lockA.tryAcquire(key, Duration.ofMillis(500));
assertTrue(a.isPresent());
// Let A's 500ms lease TTL-expire so Valkey drops the key, then B takes a fresh lock.
Thread.sleep(800);
Optional<DistributedLock.LockHandle> b = lockB.tryAcquire(key, Duration.ofSeconds(30));
assertTrue(b.isPresent(), "B must acquire after A's lease expired");
// A's stale handle (different UUID value) must touch neither B's renew nor B's key.
assertFalse(
a.get().renew(Duration.ofSeconds(30)),
"stale owner must not be able to renew a lock now owned by B");
a.get().release(); // value-checked DEL: must be a no-op, must NOT delete B's key
assertFalse(
lockA.tryAcquire(key, Duration.ofMillis(100)).isPresent(),
"B must still hold the lock; A's stale release must not have stolen it");
b.get().release();
}
@Test
@DisplayName("register is atomic (hash + TTL committed together, no orphan keys on crash)")
void registryRegisterIsAtomic() {
ValkeyInstanceRegistry reg = new ValkeyInstanceRegistry(templateA);
ClusterNode node =
new ClusterNode(
"atomic-node-" + java.util.UUID.randomUUID(),
"10.0.0.99:8080",
Instant.now(),
"BOTH");
reg.register(node, Duration.ofSeconds(30));
// TTL must be positive; -1 would mean EXPIRE did not commit inside MULTI/EXEC.
Long ttlMs =
templateA.getExpire(
"stirling:nodes:" + node.nodeId(),
java.util.concurrent.TimeUnit.MILLISECONDS);
assertNotNull(ttlMs);
assertTrue(
ttlMs > 0 && ttlMs <= 30_000,
"register() must atomically arm TTL; expected (0, 30000] ms, got " + ttlMs);
Optional<ClusterNode> seen = reg.lookup(node.nodeId());
assertTrue(seen.isPresent(), "hash fields must be visible after atomic register()");
assertEquals("10.0.0.99:8080", seen.get().internalAddress());
reg.deregister(node.nodeId());
}
@Test
@DisplayName("register on connection A is visible from connection B")
void registryCrossConnection() {
ValkeyInstanceRegistry regA = new ValkeyInstanceRegistry(templateA);
ValkeyInstanceRegistry regB = new ValkeyInstanceRegistry(templateB);
ClusterNode node = new ClusterNode("live-node-7", "10.0.0.7:8080", Instant.now(), "BOTH");
regA.register(node, Duration.ofSeconds(30));
Optional<ClusterNode> seen = regB.lookup("live-node-7");
assertTrue(seen.isPresent());
assertEquals("10.0.0.7:8080", seen.get().internalAddress());
boolean inActive =
regB.activeNodes().stream().anyMatch(n -> "live-node-7".equals(n.nodeId()));
assertTrue(inActive);
regA.deregister("live-node-7");
assertFalse(regB.lookup("live-node-7").isPresent());
}
@Test
@DisplayName("Bucket4j: no fixed-window boundary doubling (parity with in-process semantics)")
void rateLimitNoBoundaryDoubling() throws InterruptedException {
ValkeyRateLimitStore store = newRateLimitStore(factoryA);
String key = "boundary-" + java.util.UUID.randomUUID();
long capacity = 5;
Duration window = Duration.ofMillis(500);
int firstAllowed = 0;
for (int i = 0; i < 10; i++) {
if (store.tryConsume(key, capacity, window).allowed()) firstAllowed++;
}
assertEquals(capacity, firstAllowed, "must allow exactly capacity tokens initially");
Thread.sleep(window.toMillis() + 50);
int secondAllowed = 0;
long start = System.nanoTime();
for (int i = 0; i < 20 && (System.nanoTime() - start) < 20_000_000L; i++) {
if (store.tryConsume(key, capacity, window).allowed()) secondAllowed++;
}
assertTrue(
secondAllowed <= capacity,
"token-bucket must not let a fresh full capacity be consumed instantly across"
+ " the boundary; got "
+ secondAllowed);
}
private ValkeyRateLimitStore newRateLimitStore(LettuceConnectionFactory factory) {
ValkeyRateLimitStore store = new ValkeyRateLimitStore(factory);
store.initProxyManager();
return store;
}
@Test
@DisplayName("JobStore put is atomic (hash + TTL + reverse index visible together)")
void jobStorePutIsAtomic() {
ValkeyJobStore store = new ValkeyJobStore(templateA);
String jobId = "atomic-job-" + java.util.UUID.randomUUID();
String fileId = "atomic-file-" + java.util.UUID.randomUUID();
store.put(
new JobStoreEntry(
jobId,
JobStoreEntry.JobState.PENDING,
"node-A",
Instant.now(),
null,
null,
List.of(fileId),
Map.of("k", "v")),
Duration.ofSeconds(30));
assertTrue(store.exists(jobId), "hash must be visible after put");
Long jobTtl =
templateA.getExpire(
"stirling:job:" + jobId, java.util.concurrent.TimeUnit.MILLISECONDS);
assertNotNull(jobTtl);
assertTrue(jobTtl > 0, "hash must have TTL armed inside the same transaction");
assertEquals(jobId, store.findJobIdByFileId(fileId).orElse(null));
Long indexTtl =
templateA.getExpire(
"stirling:file2job:" + fileId, java.util.concurrent.TimeUnit.MILLISECONDS);
assertNotNull(indexTtl);
assertTrue(indexTtl > 0, "reverse index must also have TTL armed");
}
@Test
@DisplayName(
"JobStore.delete(): WATCH aborts when put() races between read and EXEC, no orphaned"
+ " reverse-index entries")
void jobStoreDeleteWatchRaceRetriesAndCleansUp() {
ValkeyJobStore store = new ValkeyJobStore(templateA);
String jobId = "watch-race-job-" + java.util.UUID.randomUUID();
String originalFile = "orig-file-" + java.util.UUID.randomUUID();
String newFile = "new-file-" + java.util.UUID.randomUUID();
store.put(
new JobStoreEntry(
jobId,
JobStoreEntry.JobState.RUNNING,
"node-A",
Instant.now(),
null,
null,
List.of(originalFile),
Map.of()),
Duration.ofSeconds(30));
// Simulate the race: between delete()'s WATCH read and EXEC, add a new fileId.
// The first EXEC aborts; the retry catches the new fileId and deletes both entries.
Thread mutator =
new Thread(
() -> {
try {
Thread.sleep(20);
} catch (InterruptedException ignored) {
Thread.currentThread().interrupt();
}
store.put(
new JobStoreEntry(
jobId,
JobStoreEntry.JobState.RUNNING,
"node-A",
Instant.now(),
null,
null,
List.of(originalFile, newFile),
Map.of()),
Duration.ofSeconds(30));
});
mutator.start();
store.delete(jobId);
try {
mutator.join(2000);
} catch (InterruptedException ignored) {
Thread.currentThread().interrupt();
}
boolean hashGone = !store.exists(jobId);
boolean origIndexGone = !store.findJobIdByFileId(originalFile).isPresent();
boolean newIndexGone = !store.findJobIdByFileId(newFile).isPresent();
if (hashGone) {
assertTrue(
origIndexGone,
"if hash is deleted, original reverse-index entry must also be gone");
assertTrue(
newIndexGone,
"if hash is deleted after the racing put(), the WATCH retry must catch the"
+ " new fileId and delete its reverse-index entry too");
} else {
assertEquals(jobId, store.findJobIdByFileId(originalFile).orElse(null));
assertEquals(jobId, store.findJobIdByFileId(newFile).orElse(null));
}
}
@Test
@DisplayName("JobStore.delete() removes hash AND every reverse-index entry atomically")
void jobStoreDeleteRemovesReverseIndexEntries() {
ValkeyJobStore store = new ValkeyJobStore(templateA);
String jobId = "del-atomic-job-" + java.util.UUID.randomUUID();
String fileA = "del-atomic-fileA-" + java.util.UUID.randomUUID();
String fileB = "del-atomic-fileB-" + java.util.UUID.randomUUID();
store.put(
new JobStoreEntry(
jobId,
JobStoreEntry.JobState.COMPLETE,
"node-A",
Instant.now(),
Instant.now(),
null,
List.of(fileA, fileB),
Map.of()),
Duration.ofSeconds(30));
assertTrue(store.exists(jobId));
assertEquals(jobId, store.findJobIdByFileId(fileA).orElse(null));
assertEquals(jobId, store.findJobIdByFileId(fileB).orElse(null));
store.delete(jobId);
// Both the main hash AND every reverse-index entry must be gone; dangling reverse-index
// entries would cause findJobIdByFileId() to return a deleted jobId.
assertFalse(store.exists(jobId), "main hash must be deleted");
assertFalse(
store.findJobIdByFileId(fileA).isPresent(),
"reverse-index entry for fileA must not survive delete()");
assertFalse(
store.findJobIdByFileId(fileB).isPresent(),
"reverse-index entry for fileB must not survive delete()");
assertFalse(
Boolean.TRUE.equals(templateA.hasKey("stirling:file2job:" + fileA)),
"raw reverse-index key for fileA must not survive delete()");
assertFalse(
Boolean.TRUE.equals(templateA.hasKey("stirling:file2job:" + fileB)),
"raw reverse-index key for fileB must not survive delete()");
}
@Test
@DisplayName("JobStore.all() walks the keyspace via SCAN, not KEYS")
void jobStoreAllUsesScanNonBlocking() {
ValkeyJobStore store = new ValkeyJobStore(templateA);
for (int i = 0; i < 15; i++) {
store.put(
new JobStoreEntry(
"scan-job-" + i,
JobStoreEntry.JobState.PENDING,
"node-A",
Instant.now(),
null,
null,
List.of(),
Map.of()),
Duration.ofSeconds(30));
}
long observed = store.all().stream().filter(e -> e.jobId().startsWith("scan-job-")).count();
assertTrue(
observed >= 15,
"SCAN-based all() must surface every inserted job, saw " + observed);
}
@Test
@DisplayName("Valkey unreachable yields isHealthy() = false")
void unreachableBackplaneReportsUnhealthy() {
RedisStandaloneConfiguration cfg = new RedisStandaloneConfiguration("localhost", 16400);
LettuceConnectionFactory dead = new LettuceConnectionFactory(cfg);
dead.afterPropertiesSet();
try {
StringRedisTemplate t = new StringRedisTemplate(dead);
ValkeyClusterBackplane bp = new ValkeyClusterBackplane(newProps("orphan"), t);
assertFalse(bp.isHealthy(), "isHealthy must be false when Valkey is unreachable");
} finally {
dead.destroy();
}
}
private ApplicationProperties newProps(String nodeId) {
ApplicationProperties p = new ApplicationProperties();
p.getCluster().setEnabled(true);
p.getCluster().setBackplane("valkey");
p.getCluster()
.getValkey()
.setUrl("redis://" + VALKEY.getHost() + ":" + VALKEY.getMappedPort(6379));
p.getCluster().getNode().setId(nodeId);
return p;
}
}
app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplaneTest.java
+61
View File
@@ -0,0 +1,61 @@
package stirling.software.proprietary.cluster.valkey;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import org.junit.jupiter.api.Test;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.StringRedisTemplate;
import stirling.software.common.model.ApplicationProperties;
/**
* S3 regression: {@link ValkeyClusterBackplane#isHealthy()} must route through {@code
* template.execute(...)} so the borrowed connection is always returned to the pool. Calling {@code
* getConnectionFactory().getConnection()} directly would leak the connection on every k8s liveness
* probe tick and exhaust the pool under monitoring load.
*/
class ValkeyClusterBackplaneTest {
@Test
void isHealthy_routesThroughTemplateExecute_andDoesNotTouchConnectionFactoryDirectly() {
StringRedisTemplate template = mock(StringRedisTemplate.class);
when(template.execute(any(RedisCallback.class))).thenReturn("PONG");
ApplicationProperties props = new ApplicationProperties();
props.getCluster().getNode().setId("n-1");
ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template);
assertTrue(bp.isHealthy());
verify(template, times(1)).execute(any(RedisCallback.class));
// Critical: never bypass the template's connection management.
verify(template, never()).getConnectionFactory();
}
@Test
void isHealthy_returnsFalseWhenExecuteThrows() {
StringRedisTemplate template = mock(StringRedisTemplate.class);
when(template.execute(any(RedisCallback.class))).thenThrow(new RuntimeException("boom"));
ApplicationProperties props = new ApplicationProperties();
props.getCluster().getNode().setId("n-1");
ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template);
assertFalse(bp.isHealthy());
}
@Test
void shouldRunLocalCleanup_returnsFalse_valkeyOwnsTtlEviction() {
StringRedisTemplate template = mock(StringRedisTemplate.class);
ApplicationProperties props = new ApplicationProperties();
props.getCluster().getNode().setId("n-1");
ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template);
assertFalse(bp.shouldRunLocalCleanup());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfigurationTest.java
+298
View File
@@ -0,0 +1,298 @@
package stirling.software.proprietary.cluster.valkey;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.atMost;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.springframework.data.redis.RedisSystemException;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import io.lettuce.core.RedisCommandExecutionException;
import io.lettuce.core.SslVerifyMode;
import stirling.software.proprietary.cluster.valkey.ValkeyConnectionConfiguration.Endpoint;
/**
* Verifies auth-fast-fail on WRONGPASS/NOAUTH/NOPERM: these are unrecoverable so the handshake must
* short-circuit after one attempt, not burn the 30 s retry loop.
*/
class ValkeyConnectionConfigurationTest {
@Test
@DisplayName("WRONGPASS surfaces in one attempt (no 30s retry loop)")
void wrongpass_failsImmediately_withoutRetries() throws Exception {
LettuceConnectionFactory factory = mock(LettuceConnectionFactory.class);
RedisConnection conn = mock(RedisConnection.class);
when(factory.getConnection()).thenReturn(conn);
RedisCommandExecutionException auth =
new RedisCommandExecutionException("WRONGPASS invalid username-password pair");
when(conn.ping()).thenThrow(new RedisSystemException("Error in execution", auth));
long start = System.nanoTime();
IllegalStateException ex =
assertThrows(
IllegalStateException.class,
() ->
ValkeyConnectionConfiguration.eagerHandshake(
factory, "valkey", 6379, false));
long elapsedMs = (System.nanoTime() - start) / 1_000_000;
verify(factory, times(1)).getConnection();
verify(conn, times(1)).ping();
assertTrue(
elapsedMs < 1500,
"Auth failure must short-circuit retries; elapsed=" + elapsedMs + " ms");
assertTrue(
ex.getMessage().contains("authentication failed"),
"Error message must explain the auth failure; got: " + ex.getMessage());
verify(factory, atMost(1)).destroy();
}
@Test
@DisplayName("NOAUTH surfaces in one attempt")
void noauth_failsImmediately() {
LettuceConnectionFactory factory = mock(LettuceConnectionFactory.class);
RedisConnection conn = mock(RedisConnection.class);
when(factory.getConnection()).thenReturn(conn);
when(conn.ping())
.thenThrow(
new RedisSystemException(
"Error in execution",
new RedisCommandExecutionException(
"NOAUTH Authentication required.")));
assertThrows(
IllegalStateException.class,
() -> ValkeyConnectionConfiguration.eagerHandshake(factory, "v", 6379, false));
verify(conn, times(1)).ping();
}
@Test
@DisplayName("NOPERM surfaces in one attempt")
void noperm_failsImmediately() {
LettuceConnectionFactory factory = mock(LettuceConnectionFactory.class);
RedisConnection conn = mock(RedisConnection.class);
when(factory.getConnection()).thenReturn(conn);
when(conn.ping())
.thenThrow(
new RedisSystemException(
"Error in execution",
new RedisCommandExecutionException(
"NOPERM this user has no permissions to run the 'ping'"
+ " command")));
assertThrows(
IllegalStateException.class,
() -> ValkeyConnectionConfiguration.eagerHandshake(factory, "v", 6379, false));
verify(conn, times(1)).ping();
}
@Test
@DisplayName("isAuthFailure - direct RedisCommandExecutionException with auth prefix")
void isAuthFailure_directRedisCommandExecutionException() {
assertTrue(
ValkeyConnectionConfiguration.isAuthFailure(
new RedisCommandExecutionException("WRONGPASS bad password")));
assertTrue(
ValkeyConnectionConfiguration.isAuthFailure(
new RedisCommandExecutionException("NOAUTH required")));
assertTrue(
ValkeyConnectionConfiguration.isAuthFailure(
new RedisCommandExecutionException("NOPERM denied")));
}
@Test
@DisplayName("isAuthFailure - wrapped inside RedisSystemException (production path)")
void isAuthFailure_wrappedBySpring() {
assertTrue(
ValkeyConnectionConfiguration.isAuthFailure(
new RedisSystemException(
"Error in execution",
new RedisCommandExecutionException("WRONGPASS bad password"))));
}
@Test
@DisplayName("isAuthFailure - connection errors do NOT count as auth failures")
void isAuthFailure_connectionErrorReturnsFalse() {
assertFalse(
ValkeyConnectionConfiguration.isAuthFailure(
new RedisSystemException(
"Redis connection failed",
new io.lettuce.core.RedisConnectionException(
"Connection refused"))));
assertFalse(
ValkeyConnectionConfiguration.isAuthFailure(
new IllegalStateException("Valkey PING returned 'foo' (expected PONG)")));
}
@Test
@DisplayName("bad PONG (protocol error) is not an auth failure")
void unexpectedPong_isNotAuthFailure() {
assertFalse(
ValkeyConnectionConfiguration.isAuthFailure(
new IllegalStateException("Valkey PING returned 'bar' (expected PONG)")));
}
@Test
@DisplayName("TLS on, skipCertVerification=false → useSsl + verifyPeer=FULL (default)")
void tls_defaultEnforcesFullPeerVerification() {
LettuceClientConfiguration cfg =
ValkeyConnectionConfiguration.buildClientConfiguration(true, false);
assertTrue(cfg.isUseSsl(), "TLS must be enabled");
assertSame(SslVerifyMode.FULL, cfg.getVerifyMode());
assertTrue(cfg.isVerifyPeer());
}
@Test
@DisplayName("TLS on, skipCertVerification=true → verifyPeer=NONE (dev override)")
void tls_skipCertVerificationOptOut() {
LettuceClientConfiguration cfg =
ValkeyConnectionConfiguration.buildClientConfiguration(true, true);
assertTrue(cfg.isUseSsl());
assertSame(SslVerifyMode.NONE, cfg.getVerifyMode());
}
@Test
@DisplayName("TLS off → no SSL, verify flag default (skipCertVerification ignored)")
void noTls_ignoresSkipFlag() {
LettuceClientConfiguration cfg =
ValkeyConnectionConfiguration.buildClientConfiguration(false, true);
assertFalse(cfg.isUseSsl());
}
@Nested
@DisplayName("parseUrl()")
class ParseUrl {
@Test
@DisplayName("host + explicit port, no auth, no TLS")
void hostAndPort() {
Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://valkey.internal:6380");
assertEquals("valkey.internal", e.host());
assertEquals(6380, e.port());
assertFalse(e.tls());
assertNull(e.username());
assertNull(e.password());
}
@Test
@DisplayName("missing port defaults to 6379")
void defaultPort() {
assertEquals(6379, ValkeyConnectionConfiguration.parseUrl("redis://host").port());
}
@Test
@DisplayName("rediss:// scheme selects TLS")
void redissSelectsTls() {
assertTrue(ValkeyConnectionConfiguration.parseUrl("rediss://host:6379").tls());
}
@Test
@DisplayName("user:password@ sets both credentials")
void userAndPassword() {
Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://alice:s3cret@host");
assertEquals("alice", e.username());
assertEquals("s3cret", e.password());
}
@Test
@DisplayName("empty user (:pw@) is password-only auth, username stays null")
void passwordOnlyEmptyUser() {
Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://:s3cret@host");
assertNull(e.username(), "empty user must NOT become an empty-string username");
assertEquals("s3cret", e.password());
}
@Test
@DisplayName("single userinfo token (no colon) is treated as the password")
void passwordOnlyNoColon() {
Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://s3cret@host");
assertNull(e.username());
assertEquals("s3cret", e.password());
}
@Test
@DisplayName("only the first colon splits user/password (colons allowed in password)")
void colonInPassword() {
Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://user:pa:ss:word@host");
assertEquals("user", e.username());
assertEquals("pa:ss:word", e.password());
}
@Test
@DisplayName("percent-encoded reserved chars are decoded in the password")
void percentEncodedPassword() {
// %40 -> '@', %23 -> '#': both must be encoded or URI parses them structurally.
Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://:p%40ss%23word@host");
assertNull(e.username());
assertEquals("p@ss#word", e.password());
}
@Test
@DisplayName("blank url throws with a backplane-config message")
void blankUrlThrows() {
IllegalStateException ex =
assertThrows(
IllegalStateException.class,
() -> ValkeyConnectionConfiguration.parseUrl(" "));
assertTrue(ex.getMessage().contains("cluster.valkey.url must be set"));
}
@Test
@DisplayName("null url throws")
void nullUrlThrows() {
assertThrows(
IllegalStateException.class,
() -> ValkeyConnectionConfiguration.parseUrl(null));
}
@Test
@DisplayName("url with no host throws a clear error (scheme-less host:port pitfall)")
void noHostThrows() {
// "localhost:6379" parses 'localhost' as the scheme, leaving no authority/host.
IllegalStateException ex =
assertThrows(
IllegalStateException.class,
() -> ValkeyConnectionConfiguration.parseUrl("localhost:6379"));
assertTrue(
ex.getMessage().contains("has no host"),
"message must name the missing host; got: " + ex.getMessage());
}
@Test
@DisplayName("non-numeric port yields no host (URI registry-authority fallback)")
void nonNumericPortHasNoHost() {
// java.net.URI does not throw on a bad port; it falls back to registry authority and
// reports host=null, so this must surface as the clear no-host error, not a NPE later.
IllegalStateException ex =
assertThrows(
IllegalStateException.class,
() -> ValkeyConnectionConfiguration.parseUrl("redis://host:notaport"));
assertTrue(ex.getMessage().contains("has no host"));
}
@Test
@DisplayName("syntactically invalid uri throws with the offending url")
void invalidUriThrows() {
IllegalStateException ex =
assertThrows(
IllegalStateException.class,
() -> ValkeyConnectionConfiguration.parseUrl("redis://ho st:6379"));
assertTrue(ex.getMessage().contains("not a valid URI"));
assertTrue(ex.getMessage().contains("redis://ho st:6379"));
}
}
}