Add Valkey cluster backplane and sticky-410 ownership (clusters) (#6472)

2026-07-14 10:34:06 +02:00 · 2026-06-02 14:59:20 +01:00
parent de9d6ad3f5
commit b355ccec9e
30 changed files with 3858 additions and 213 deletions
@@ -4,6 +4,7 @@ import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;

 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.MediaType;
@@ -22,6 +23,10 @@ import jakarta.servlet.http.HttpServletRequest;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.StickyMissRecorder;
 import stirling.software.common.model.job.JobResult;
 import stirling.software.common.model.job.ResultFile;
 import stirling.software.common.service.FileStorage;
@@ -30,7 +35,6 @@ import stirling.software.common.service.JobQueue;
 import stirling.software.common.service.TaskManager;
 import stirling.software.common.util.RegexPatternUtils;

-/** REST controller for job-related endpoints */
@RestController
@RequiredArgsConstructor
@Slf4j
@@ -42,20 +46,29 @@ public class JobController {
    private final FileStorage fileStorage;
    private final JobQueue jobQueue;
    private final HttpServletRequest request;
+    private final ClusterBackplane clusterBackplane;
+    private final JobStore jobStore;
+
+    // Short-TTL local cache fronting JobStore.get() on the sticky-410 path to avoid a Valkey
+    // HGETALL round-trip on every download retry for the same job.
+    private final JobOwnershipCache ownershipCache = new JobOwnershipCache();

    @Autowired(required = false)
    private JobOwnershipService jobOwnershipService;

-    /**
-     * Get the status of a job
-     *
-     * @param jobId The job ID
-     * @return The job result
-     */
+    @Autowired(required = false)
+    private StickyMissRecorder stickyMissRecorder;
+
    @GetMapping("/job/{jobId}")
    @Operation(summary = "Get job status")
    public ResponseEntity<?> getJobStatus(@PathVariable("jobId") String jobId) {
-        // Validate job ownership
+        // Sticky-410 must run before user-auth: a 403 here would leak job existence and defeat
+        // LB re-routing. The owner node is where the real auth check should happen.
+        Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
+        if (peerOwned.isPresent()) {
+            return peerOwned.get();
+        }
+
        if (!validateJobAccess(jobId)) {
            log.warn("Unauthorized attempt to access job status: {}", jobId);
            return ResponseEntity.status(403)
@@ -67,7 +80,6 @@ public class JobController {
            return ResponseEntity.notFound().build();
        }

-        // Check if the job is in the queue and add queue information
        if (!result.isComplete() && jobQueue.isJobQueued(jobId)) {
            int position = jobQueue.getJobPosition(jobId);
            Map<String, Object> resultWithQueueInfo =
@@ -82,16 +94,14 @@ public class JobController {
        return ResponseEntity.ok(result);
    }

-    /**
-     * Get the result of a job
-     *
-     * @param jobId The job ID
-     * @return The job result
-     */
    @GetMapping("/job/{jobId}/result")
    @Operation(summary = "Get job result")
    public ResponseEntity<?> getJobResult(@PathVariable("jobId") String jobId) {
-        // Validate job ownership
+        Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
+        if (peerOwned.isPresent()) {
+            return peerOwned.get();
+        }
+
        if (!validateJobAccess(jobId)) {
            log.warn("Unauthorized attempt to access job result: {}", jobId);
            return ResponseEntity.status(403)
@@ -111,7 +121,6 @@ public class JobController {
            return ResponseEntity.badRequest().body("Job failed: " + result.getError());
        }

-        // Handle multiple files - return metadata for client to download individually
        if (result.hasMultipleFiles()) {
            return ResponseEntity.ok()
                    .contentType(MediaType.APPLICATION_JSON)
@@ -125,11 +134,11 @@ public class JobController {
                                    result.getAllResultFiles()));
        }

-        // Handle single file (download directly)
        if (result.hasFiles() && !result.hasMultipleFiles()) {
            try {
                List<ResultFile> files = result.getAllResultFiles();
                ResultFile singleFile = files.get(0);
+
                byte[] fileContent = fileStorage.retrieveBytes(singleFile.getFileId());
                return ResponseEntity.ok()
                        .header("Content-Type", singleFile.getContentType())
@@ -147,30 +156,22 @@ public class JobController {
        return ResponseEntity.ok(result.getResult());
    }

-    // Admin-only endpoints have been moved to AdminJobController in the proprietary package
-
-    /**
-     * Cancel a job by its ID
-     *
-     * <p>This method should only allow cancellation of jobs that were created by the current user.
-     * The jobId should be part of the user's session or otherwise linked to their identity.
-     *
-     * @param jobId The job ID
-     * @return Response indicating whether the job was cancelled
-     */
    @DeleteMapping("/job/{jobId}")
    @Operation(summary = "Cancel a job")
    public ResponseEntity<?> cancelJob(@PathVariable("jobId") String jobId) {
        log.debug("Request to cancel job: {}", jobId);

-        // Validate job ownership
+        Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
+        if (peerOwned.isPresent()) {
+            return peerOwned.get();
+        }
+
        if (!validateJobAccess(jobId)) {
            log.warn("Unauthorized attempt to cancel job: {}", jobId);
            return ResponseEntity.status(403)
                    .body(Map.of("message", "You are not authorized to cancel this job"));
        }

-        // First check if the job is in the queue
        boolean cancelled = false;
        int queuePosition = -1;

@@ -180,11 +181,9 @@ public class JobController {
            log.info("Cancelled queued job: {} (was at position {})", jobId, queuePosition);
        }

-        // If not in queue or couldn't cancel, try to cancel in TaskManager
        if (!cancelled) {
            JobResult result = taskManager.getJobResult(jobId);
            if (result != null && !result.isComplete()) {
-                // Mark as error with cancellation message
                taskManager.setError(jobId, "Job was cancelled by user");
                cancelled = true;
                log.info("Marked job as cancelled in TaskManager: {}", jobId);
@@ -201,7 +200,6 @@ public class JobController {
                            "queuePosition",
                            queuePosition >= 0 ? queuePosition : "n/a"));
        } else {
-            // Job not found or already complete
            JobResult result = taskManager.getJobResult(jobId);
            if (result == null) {
                return ResponseEntity.notFound().build();
@@ -215,16 +213,14 @@ public class JobController {
        }
    }

-    /**
-     * Get the list of files for a job
-     *
-     * @param jobId The job ID
-     * @return List of files for the job
-     */
    @GetMapping("/job/{jobId}/result/files")
    @Operation(summary = "Get job result files")
    public ResponseEntity<?> getJobFiles(@PathVariable("jobId") String jobId) {
-        // Validate job ownership
+        Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
+        if (peerOwned.isPresent()) {
+            return peerOwned.get();
+        }
+
        if (!validateJobAccess(jobId)) {
            log.warn("Unauthorized attempt to access job files: {}", jobId);
            return ResponseEntity.status(403)
@@ -252,28 +248,31 @@ public class JobController {
                        "files", files));
    }

-    /**
-     * Get metadata for an individual file by its file ID
-     *
-     * @param fileId The file ID
-     * @return The file metadata
-     */
    @GetMapping("/files/{fileId}/metadata")
    @Operation(summary = "Get file metadata")
    public ResponseEntity<?> getFileMetadata(@PathVariable("fileId") String fileId) {
        try {
-            String jobKey = taskManager.findJobKeyByFileId(fileId);
+            String jobKey;
+            try {
+                jobKey = taskManager.findJobKeyByFileId(fileId);
+            } catch (RuntimeException backplaneEx) {
+                return backplaneUnavailable(fileId, backplaneEx);
+            }
            if (jobKey == null) {
                return ResponseEntity.notFound().build();
            }

+            Optional<ResponseEntity<?>> notOwner = guardNonOwner(jobKey);
+            if (notOwner.isPresent()) {
+                return notOwner.get();
+            }
+
            if (!validateJobAccess(jobKey)) {
                log.warn("Unauthorized attempt to access file metadata: {}", fileId);
                return ResponseEntity.status(403)
                        .body(Map.of("message", "You are not authorized to access this file"));
            }

-            // Find the file metadata from any job that contains this file
            ResultFile resultFile = taskManager.findResultFileByFileId(fileId);

            if (resultFile != null) {
@@ -281,12 +280,10 @@ public class JobController {
            }

            if (!isSecurityEnabled()) {
-                // Backwards compatibility when ownership service is unavailable
                if (!fileStorage.fileExists(fileId)) {
                    return ResponseEntity.notFound().build();
                }

-                // File exists but no metadata found, get basic info efficiently
                long fileSize = fileStorage.getFileSize(fileId);
                return ResponseEntity.ok(
                        Map.of(
@@ -308,32 +305,31 @@ public class JobController {
        }
    }

-    /**
-     * Download an individual file by its file ID
-     *
-     * @param fileId The file ID
-     * @return The file content
-     */
    @GetMapping("/files/{fileId}")
    @Operation(summary = "Download a file")
    public ResponseEntity<?> downloadFile(@PathVariable("fileId") String fileId) {
        try {
-            String jobKey = taskManager.findJobKeyByFileId(fileId);
+            String jobKey;
+            try {
+                jobKey = taskManager.findJobKeyByFileId(fileId);
+            } catch (RuntimeException backplaneEx) {
+                return backplaneUnavailable(fileId, backplaneEx);
+            }
            if (jobKey == null) {
                return ResponseEntity.notFound().build();
            }

+            Optional<ResponseEntity<?>> notOwner = guardNonOwner(jobKey);
+            if (notOwner.isPresent()) {
+                return notOwner.get();
+            }
+
            if (!validateJobAccess(jobKey)) {
                log.warn("Unauthorized attempt to download file: {}", fileId);
                return ResponseEntity.status(403)
                        .body(Map.of("message", "You are not authorized to access this file"));
            }

-            // Retrieve file content
-            byte[] fileContent = fileStorage.retrieveBytes(fileId);
-
-            // Find the file metadata from any job that contains this file
-            // This is for getting the original filename and content type
            ResultFile resultFile = taskManager.findResultFileByFileId(fileId);

            String fileName = resultFile != null ? resultFile.getFileName() : "download";
@@ -342,6 +338,8 @@ public class JobController {
                            ? resultFile.getContentType()
                            : MediaType.APPLICATION_OCTET_STREAM_VALUE;

+            byte[] fileContent = fileStorage.retrieveBytes(fileId);
+
            return ResponseEntity.ok()
                    .header("Content-Type", contentType)
                    .header("Content-Disposition", createContentDispositionHeader(fileName))
@@ -357,11 +355,88 @@ public class JobController {
    }

    /**
-     * Create Content-Disposition header with UTF-8 filename support
-     *
-     * @param fileName The filename to encode
-     * @return Content-Disposition header value
+     * Returns 410 Gone when the job is owned by a peer node, empty otherwise. Uses a short-TTL
+     * local cache to avoid repeated Valkey lookups on the hot download path. When the backplane is
+     * unreachable, a locally-held job is still served and anything else gets a retryable 503.
     */
+    private Optional<ResponseEntity<?>> guardNonOwner(String jobId) {
+        if (clusterBackplane == null || jobStore == null) {
+            return Optional.empty();
+        }
+        Optional<JobStoreEntry> entry;
+        Optional<Optional<JobStoreEntry>> cached = ownershipCache.get(jobId);
+        if (cached.isPresent()) {
+            entry = cached.get();
+        } else {
+            try {
+                entry = jobStore.get(jobId);
+            } catch (RuntimeException ex) {
+                // Backplane unreachable: if we hold the job locally serve it, otherwise return a
+                // retryable 503 (same contract as the file endpoints) instead of a misleading 404.
+                if (taskManager.getJobResult(jobId) == null) {
+                    return Optional.of(backplaneUnavailable(jobId, ex));
+                }
+                log.warn(
+                        "JobStore lookup failed for jobId={}; serving locally-held job: {}",
+                        jobId,
+                        ex.getMessage());
+                return Optional.empty();
+            }
+            ownershipCache.put(jobId, entry);
+        }
+        if (entry.isEmpty()) {
+            return Optional.empty();
+        }
+        String owner = entry.get().owningNodeId();
+        if (owner == null || owner.isBlank()) {
+            return Optional.empty();
+        }
+        String localId = clusterBackplane.localNodeId();
+        if (owner.equals(localId)) {
+            return Optional.empty();
+        }
+        log.info(
+                "Sticky-session miss for jobId={} (owner={}, local={}); returning 410 so client"
+                        + " retries via LB affinity",
+                jobId,
+                owner,
+                localId);
+        if (stickyMissRecorder != null) {
+            stickyMissRecorder.recordStickyMiss();
+        }
+        return Optional.of(
+                ResponseEntity.status(410)
+                        .header("Retry-After", "0")
+                        .body(
+                                Map.of(
+                                        "message",
+                                        "Result lives on another node. Retry to be routed there"
+                                                + " by the load balancer's sticky-session"
+                                                + " affinity, or re-run the job.",
+                                        "ownedBy",
+                                        owner,
+                                        "currentNode",
+                                        localId == null ? "" : localId)));
+    }
+
+    /**
+     * When the backplane is unreachable we cannot resolve ownership or existence, and serving
+     * without that check would be unsafe - so return a retryable 503 (consistent with the
+     * sticky-410 retry model) rather than a misleading 404 or a generic 500.
+     */
+    private ResponseEntity<?> backplaneUnavailable(String id, RuntimeException ex) {
+        log.warn(
+                "Backplane lookup failed for {}; returning 503 (retryable): {}",
+                id,
+                ex.getMessage());
+        return ResponseEntity.status(503)
+                .header("Retry-After", "1")
+                .body(
+                        Map.of(
+                                "message",
+                                "Cluster backplane temporarily unavailable; retry shortly."));
+    }
+
    private String createContentDispositionHeader(String fileName) {
        try {
            String encodedFileName =
@@ -371,19 +446,11 @@ public class JobController {
                            .replaceAll("%20"); // URLEncoder uses + for spaces, but we want %20
            return "attachment; filename=\"" + fileName + "\"; filename*=UTF-8''" + encodedFileName;
        } catch (Exception e) {
-            // Fallback to basic filename if encoding fails
            return "attachment; filename=\"" + fileName + "\"";
        }
    }

-    /**
-     * Validate that the current user has access to the given job.
-     *
-     * @param jobId the job identifier to validate
-     * @return true if user has access, false otherwise
-     */
    private boolean validateJobAccess(String jobId) {
-        // If JobOwnershipService is available (security enabled), use it
        if (jobOwnershipService != null) {
            try {
                return jobOwnershipService.validateJobAccess(jobId);
@@ -393,8 +460,6 @@ public class JobController {
            }
        }

-        // Security disabled - allow all access (backwards compatibility)
-        // When security is not enabled, any user can access any job by jobId
        return true;
    }
 }
@@ -0,0 +1,47 @@
+package stirling.software.common.controller;
+
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import stirling.software.common.cluster.JobStoreEntry;
+
+/**
+ * Process-local TTL cache for {@link JobStoreEntry} lookups to suppress redundant Valkey HGETALL
+ * round-trips on the hot result-download path (sticky-410 ownership check).
+ *
+ * <p>5 second TTL is short enough that a job's lifecycle transitions (RUNNING -> COMPLETE -> TTL
+ * expiry) propagate to all nodes within the LB's sticky-session window, and short enough that a
+ * mistakenly-cached "not found" recovers quickly when an entry actually shows up. Cap the map at
+ * 2048 entries to bound memory; eviction is best-effort (clear-and-restart) since the cache is
+ * advisory.
+ */
+final class JobOwnershipCache {
+
+    private static final long TTL_NANOS = 5L * 1_000_000_000L; // 5 s
+    private static final int MAX_ENTRIES = 2048;
+
+    private final ConcurrentMap<String, Entry> entries = new ConcurrentHashMap<>();
+
+    Optional<Optional<JobStoreEntry>> get(String jobId) {
+        Entry e = entries.get(jobId);
+        if (e == null) {
+            return Optional.empty();
+        }
+        if (System.nanoTime() - e.storedAtNanos > TTL_NANOS) {
+            entries.remove(jobId, e);
+            return Optional.empty();
+        }
+        return Optional.of(e.value);
+    }
+
+    void put(String jobId, Optional<JobStoreEntry> value) {
+        if (entries.size() >= MAX_ENTRIES) {
+            // Best-effort eviction; under burst the cache simply rebuilds.
+            entries.clear();
+        }
+        entries.put(jobId, new Entry(value, System.nanoTime()));
+    }
+
+    private record Entry(Optional<JobStoreEntry> value, long storedAtNanos) {}
+}
@@ -0,0 +1,479 @@
+package stirling.software.common.controller;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import jakarta.servlet.http.HttpServletRequest;
+
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.StickyMissRecorder;
+import stirling.software.common.model.job.JobResult;
+import stirling.software.common.service.FileStorage;
+import stirling.software.common.service.JobOwnershipService;
+import stirling.software.common.service.JobQueue;
+import stirling.software.common.service.TaskManager;
+
+/**
+ * Sticky-410 ownership contract for {@link JobController}: peer-owned jobs return 410 Gone with
+ * ownedBy/currentNode fields; locally-owned and no-entry cases return 200. FileStorage is never
+ * touched on the 410 path. Manual mock construction so tests can vary backplane/jobstore combos.
+ */
+class JobControllerOwnershipTest {
+
+    private TaskManager taskManager;
+    private FileStorage fileStorage;
+    private JobQueue jobQueue;
+    private HttpServletRequest request;
+    private JobOwnershipService jobOwnershipService;
+    private ClusterBackplane clusterBackplane;
+    private JobStore jobStore;
+    private StickyMissRecorder stickyMissRecorder;
+
+    private static final String JOB_ID = "job-42";
+    private static final String FILE_ID = "file-abc";
+    private static final String LOCAL_NODE = "node-self";
+    private static final String PEER_NODE = "node-peer";
+
+    @BeforeEach
+    void setUp() {
+        taskManager = mock(TaskManager.class);
+        fileStorage = mock(FileStorage.class);
+        jobQueue = mock(JobQueue.class);
+        request = mock(HttpServletRequest.class);
+        jobOwnershipService = mock(JobOwnershipService.class);
+        clusterBackplane = mock(ClusterBackplane.class);
+        jobStore = mock(JobStore.class);
+        stickyMissRecorder = mock(StickyMissRecorder.class);
+    }
+
+    private JobController makeController(ClusterBackplane backplane, JobStore store) {
+        JobController c =
+                new JobController(taskManager, fileStorage, jobQueue, request, backplane, store);
+        ReflectionTestUtils.setField(c, "stickyMissRecorder", stickyMissRecorder);
+        return c;
+    }
+
+    private JobController makeController() {
+        return makeController(clusterBackplane, jobStore);
+    }
+
+    private JobStoreEntry entryOwnedBy(String ownerNodeId) {
+        return new JobStoreEntry(
+                JOB_ID,
+                JobStoreEntry.JobState.COMPLETE,
+                ownerNodeId,
+                Instant.now(),
+                Instant.now(),
+                null,
+                List.of(FILE_ID),
+                Map.of());
+    }
+
+    private JobResult completedJobWithFile() {
+        JobResult result = new JobResult();
+        result.setJobId(JOB_ID);
+        // completeWithSingleFile populates the resultFiles list, sets complete=true,
+        // and sets completedAt - all required for the getJobResult single-file branch.
+        result.completeWithSingleFile(FILE_ID, "out.pdf", "application/pdf", 7L);
+        return result;
+    }
+
+    @Test
+    @DisplayName(
+            "downloadFile peer-owned → full sticky-410 contract"
+                    + " (status + Retry-After + payload + metric + storage untouched)")
+    void downloadFile_peerOwned_fullStickyContract() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+
+        ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        assertEquals("0", response.getHeaders().getFirst("Retry-After"));
+
+        assertInstanceOf(Map.class, response.getBody());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals(3, body.size(), "exactly: message, ownedBy, currentNode");
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+        assertEquals(LOCAL_NODE, body.get("currentNode"));
+        assertNotNull(body.get("message"));
+        assertTrue(((String) body.get("message")).toLowerCase().contains("retry"));
+        assertNull(body.get("internalSecret"));
+        assertNull(body.get("filePath"));
+        verify(stickyMissRecorder).recordStickyMiss();
+        verify(fileStorage, never()).retrieveBytes(FILE_ID);
+    }
+
+    private static Stream<Arguments> downloadHappyPathScenarios() {
+        return Stream.of(
+                Arguments.of("locallyOwned", LOCAL_NODE, true),
+                Arguments.of("noJobStoreEntry", null, false),
+                Arguments.of("blankOwningNodeId", "", true));
+    }
+
+    @ParameterizedTest(name = "downloadFile {0} -> 200, no sticky-miss")
+    @MethodSource("downloadHappyPathScenarios")
+    void downloadFile_happyPath_returnsOkAndNoMetric(
+            String scenario, String ownerNodeId, boolean entryPresent) throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID))
+                .thenReturn(
+                        entryPresent ? Optional.of(entryOwnedBy(ownerNodeId)) : Optional.empty());
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode(), scenario);
+        verify(fileStorage).retrieveBytes(FILE_ID);
+        verify(stickyMissRecorder, never()).recordStickyMiss();
+    }
+
+    @Test
+    @DisplayName("getJobResult: locally-owned single-file result → reads from FileStorage, 200 OK")
+    void getJobResult_singleFile_locallyOwned_readsFromStorage() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        ResponseEntity<?> response = makeController().getJobResult(JOB_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+    }
+
+    private enum Endpoint {
+        DOWNLOAD_FILE,
+        GET_JOB_RESULT,
+        GET_JOB_STATUS,
+        GET_JOB_FILES,
+        GET_FILE_METADATA,
+        CANCEL_JOB
+    }
+
+    private static Stream<Arguments> peerOwned410Scenarios() {
+        return Stream.of(
+                Arguments.of(Endpoint.DOWNLOAD_FILE),
+                Arguments.of(Endpoint.GET_JOB_RESULT),
+                Arguments.of(Endpoint.GET_JOB_STATUS),
+                Arguments.of(Endpoint.GET_JOB_FILES),
+                Arguments.of(Endpoint.GET_FILE_METADATA),
+                Arguments.of(Endpoint.CANCEL_JOB));
+    }
+
+    @ParameterizedTest(name = "{0} peer-owned -> 410, ownedBy=peer, metric++")
+    @MethodSource("peerOwned410Scenarios")
+    void endpoint_peerOwned_returns410(Endpoint endpoint) throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+        switch (endpoint) {
+            case DOWNLOAD_FILE, GET_FILE_METADATA ->
+                    when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+            case GET_JOB_RESULT ->
+                    when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
+            case GET_JOB_STATUS, GET_JOB_FILES ->
+                    when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
+            case CANCEL_JOB -> {
+                when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
+                when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
+            }
+        }
+
+        ResponseEntity<?> response =
+                switch (endpoint) {
+                    case DOWNLOAD_FILE -> makeController().downloadFile(FILE_ID);
+                    case GET_JOB_RESULT -> makeController().getJobResult(JOB_ID);
+                    case GET_JOB_STATUS -> makeController().getJobStatus(JOB_ID);
+                    case GET_JOB_FILES -> makeController().getJobFiles(JOB_ID);
+                    case GET_FILE_METADATA -> makeController().getFileMetadata(FILE_ID);
+                    case CANCEL_JOB -> makeController().cancelJob(JOB_ID);
+                };
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+        assertEquals(LOCAL_NODE, body.get("currentNode"));
+        verify(stickyMissRecorder).recordStickyMiss();
+        verify(fileStorage, never()).retrieveBytes(FILE_ID);
+        if (endpoint == Endpoint.CANCEL_JOB) {
+            verify(taskManager, never()).setError(JOB_ID, "Job was cancelled by user");
+        }
+    }
+
+    private static Stream<Arguments> unknownJob404Scenarios() {
+        return Stream.of(Arguments.of(Endpoint.GET_JOB_STATUS), Arguments.of(Endpoint.CANCEL_JOB));
+    }
+
+    @ParameterizedTest(name = "{0} unknown jobId -> 404 (not 410), no metric")
+    @MethodSource("unknownJob404Scenarios")
+    void endpoint_unknownJob_returns404(Endpoint endpoint) {
+        when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.empty());
+        if (endpoint == Endpoint.CANCEL_JOB) {
+            when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
+        }
+
+        ResponseEntity<?> response =
+                switch (endpoint) {
+                    case GET_JOB_STATUS -> makeController().getJobStatus(JOB_ID);
+                    case CANCEL_JOB -> makeController().cancelJob(JOB_ID);
+                    default -> throw new IllegalArgumentException(endpoint.name());
+                };
+
+        assertEquals(HttpStatus.NOT_FOUND, response.getStatusCode());
+        verify(stickyMissRecorder, never()).recordStickyMiss();
+    }
+
+    @Test
+    @DisplayName("Single-instance install (no ClusterBackplane bean): no 410, no NPE")
+    void singleInstance_noClusterBackplane_noGoneResponse() throws Exception {
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        ResponseEntity<?> response = makeController(null, jobStore).downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+        verify(fileStorage).retrieveBytes(FILE_ID);
+    }
+
+    @Test
+    @DisplayName("Single-instance install (no JobStore bean): no 410, no NPE")
+    void singleInstance_noJobStore_noGoneResponse() throws Exception {
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        ResponseEntity<?> response = makeController(clusterBackplane, null).downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+        verify(fileStorage).retrieveBytes(FILE_ID);
+    }
+
+    @Test
+    @DisplayName("Single-instance (no StickyMissRecorder bean) → no NPE, still 200 OK")
+    void noStickyMissRecorder_works() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        JobController c = makeController();
+        ReflectionTestUtils.setField(c, "stickyMissRecorder", null);
+
+        ResponseEntity<?> response = c.downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+    }
+
+    @Test
+    @DisplayName(
+            "cluster-mode but localNodeId is null → no NPE; 410 because owner is set and"
+                    + " differs from blank")
+    void clusterBackplanePresent_butLocalNodeIdNull_falsBackGracefully() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(null);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+
+        // We still 410: owner is "node-peer", local is null → they don't match. Rather than
+        // silently 200-from-wrong-disk (which would serve garbage), we surface the mismatch.
+        ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals("", body.get("currentNode"), "blank when localNodeId is null");
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+    }
+
+    @Test
+    @DisplayName("Owner returns 410 even when JobOwnershipService allows access (orthogonal)")
+    void ownershipService_passes_butStickyStillReturns410() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+        lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(true);
+
+        JobController c = makeController();
+        ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
+        ResponseEntity<?> response = c.downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+    }
+
+    @Test
+    @DisplayName(
+            "downloadFile: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak"
+                    + " file existence")
+    void downloadFile_peerOwned_ownershipDenied_returns410NotForbidden() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+        lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
+
+        JobController c = makeController();
+        ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
+        ResponseEntity<?> response = c.downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+        verify(fileStorage, never()).retrieveBytes(FILE_ID);
+    }
+
+    @Test
+    @DisplayName(
+            "getJobStatus: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak"
+                    + " job existence")
+    void getJobStatus_peerOwned_ownershipDenied_returns410NotForbidden() {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+        lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
+
+        JobController c = makeController();
+        ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
+        ResponseEntity<?> response = c.getJobStatus(JOB_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+    }
+
+    @Test
+    @DisplayName(
+            "cancelJob: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak job"
+                    + " existence")
+    void cancelJob_peerOwned_ownershipDenied_returns410NotForbidden() {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+        lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
+
+        JobController c = makeController();
+        ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
+        ResponseEntity<?> response = c.cancelJob(JOB_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+        verify(taskManager, never()).setError(JOB_ID, "Job was cancelled by user");
+    }
+
+    @Test
+    @DisplayName(
+            "guardNonOwner caches JobStore.get within TTL window: second call same jobId hits"
+                    + " cache, not Valkey")
+    void guardNonOwner_cachesJobStoreLookupWithinTtl() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        JobController c = makeController();
+        c.downloadFile(FILE_ID);
+        c.downloadFile(FILE_ID);
+        c.downloadFile(FILE_ID);
+
+        verify(jobStore, times(1)).get(JOB_ID);
+    }
+
+    @Test
+    @DisplayName(
+            "guardNonOwner: JobStore.get throws (Valkey timeout) → falls through to local-disk"
+                    + " path, no 500 leaks to caller")
+    void guardNonOwner_jobStoreException_fallsThroughToLocalPath() throws Exception {
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout"));
+        when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+        verify(fileStorage).retrieveBytes(FILE_ID);
+        verify(stickyMissRecorder, never()).recordStickyMiss();
+    }
+
+    @Test
+    @DisplayName("backplane down + job NOT held locally → 503 retryable (not a misleading 404)")
+    void jobEndpoint_backplaneDown_notLocal_returns503() {
+        when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout"));
+        when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
+
+        ResponseEntity<?> response = makeController().getJobStatus(JOB_ID);
+
+        assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
+        assertEquals("1", response.getHeaders().getFirst("Retry-After"));
+        verify(stickyMissRecorder, never()).recordStickyMiss();
+    }
+
+    @Test
+    @DisplayName("backplane down but job held locally → owner still serves (not 503)")
+    void jobEndpoint_backplaneDown_local_servesLocally() {
+        when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout"));
+        when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
+
+        ResponseEntity<?> response = makeController().getJobStatus(JOB_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+    }
+
+    @Test
+    @DisplayName(
+            "downloadFile: findJobKeyByFileId throws (backplane down) → 503 + Retry-After,"
+                    + " not 404/500, storage untouched")
+    void downloadFile_findJobKeyThrows_returns503Retryable() throws Exception {
+        when(taskManager.findJobKeyByFileId(FILE_ID))
+                .thenThrow(new RuntimeException("Valkey command timeout"));
+
+        ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
+        assertEquals("1", response.getHeaders().getFirst("Retry-After"));
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertTrue(((String) body.get("message")).toLowerCase().contains("unavailable"));
+        verify(fileStorage, never()).retrieveBytes(FILE_ID);
+    }
+
+    @Test
+    @DisplayName(
+            "getFileMetadata: findJobKeyByFileId throws (backplane down) → 503 + Retry-After,"
+                    + " not 404/500")
+    void getFileMetadata_findJobKeyThrows_returns503Retryable() throws Exception {
+        when(taskManager.findJobKeyByFileId(FILE_ID))
+                .thenThrow(new RuntimeException("Valkey command timeout"));
+
+        ResponseEntity<?> response = makeController().getFileMetadata(FILE_ID);
+
+        assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
+        assertEquals("1", response.getHeaders().getFirst("Retry-After"));
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertTrue(((String) body.get("message")).toLowerCase().contains("unavailable"));
+        verify(fileStorage, never()).retrieveBytes(FILE_ID);
+    }
+}
@@ -19,6 +19,8 @@ import org.springframework.test.util.ReflectionTestUtils;

 import jakarta.servlet.http.HttpServletRequest;

+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.JobStore;
 import stirling.software.common.model.job.JobResult;
 import stirling.software.common.service.FileStorage;
 import stirling.software.common.service.JobOwnershipService;
@@ -37,6 +39,10 @@ class JobControllerTest {

    @Mock private JobOwnershipService jobOwnershipService;

+    @Mock private ClusterBackplane clusterBackplane;
+
+    @Mock private JobStore jobStore;
+
    private MockHttpSession session;

    @InjectMocks private JobController controller;