Add S3 storage and cluster artifact backend (#6457)

This commit is contained in:
Anthony Stirling
2026-05-28 13:06:27 +01:00
committed by GitHub
parent 57af5b9dc2
commit b3c4b8b463
25 changed files with 3183 additions and 7 deletions
@@ -0,0 +1,200 @@
package stirling.software.proprietary.cluster.s3;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3ClientBuilder;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
/**
* Shared factory for {@link S3Client} and {@link S3Presigner} instances used by both {@code
* S3StorageProvider} and {@code S3FileStore}, so endpoint/region/credentials wiring lives in
* exactly one place.
*/
@Slf4j
public final class S3Clients {
private S3Clients() {}
/** Paired client and presigner with coordinated lifecycle. */
public record Bundle(S3Client client, S3Presigner presigner) implements AutoCloseable {
@Override
public void close() {
try {
presigner.close();
} catch (Exception e) {
log.warn("Error closing S3 presigner", e);
}
try {
client.close();
} catch (Exception e) {
log.warn("Error closing S3 client", e);
}
}
}
/** Build a client+presigner pair from the shared S3 config block. */
public static Bundle build(ApplicationProperties.Storage.S3 cfg, String usage) {
if (cfg == null) {
throw new IllegalStateException(
usage + " requires storage.s3.* configuration to be set");
}
if (cfg.getBucket() == null || cfg.getBucket().isBlank()) {
throw new IllegalStateException(usage + " requires storage.s3.bucket to be set");
}
String region =
cfg.getRegion() == null || cfg.getRegion().isBlank()
? "us-east-1"
: cfg.getRegion();
S3Configuration s3Configuration =
S3Configuration.builder().pathStyleAccessEnabled(cfg.isPathStyleAccess()).build();
RequestChecksumCalculation requestChecksum =
parseRequestChecksum(cfg.getRequestChecksumCalculation());
ResponseChecksumValidation responseChecksum =
parseResponseChecksum(cfg.getResponseChecksumValidation());
S3ClientBuilder clientBuilder =
S3Client.builder()
.httpClient(UrlConnectionHttpClient.create())
.region(Region.of(region))
.serviceConfiguration(s3Configuration)
.requestChecksumCalculation(requestChecksum)
.responseChecksumValidation(responseChecksum);
S3Presigner.Builder presignerBuilder =
S3Presigner.builder()
.region(Region.of(region))
.serviceConfiguration(s3Configuration);
if (cfg.getEndpoint() != null && !cfg.getEndpoint().isBlank()) {
URI endpoint;
try {
endpoint = new URI(cfg.getEndpoint());
} catch (URISyntaxException e) {
throw new IllegalStateException(
"Invalid storage.s3.endpoint: " + cfg.getEndpoint(), e);
}
validateEndpointHost(endpoint, cfg.isAllowPrivateEndpoints());
clientBuilder.endpointOverride(endpoint);
presignerBuilder.endpointOverride(endpoint);
}
boolean hasStaticCreds =
cfg.getAccessKey() != null
&& !cfg.getAccessKey().isBlank()
&& cfg.getSecretKey() != null
&& !cfg.getSecretKey().isBlank();
if (hasStaticCreds) {
AwsBasicCredentials credentials =
AwsBasicCredentials.create(cfg.getAccessKey(), cfg.getSecretKey());
StaticCredentialsProvider provider = StaticCredentialsProvider.create(credentials);
clientBuilder.credentialsProvider(provider);
presignerBuilder.credentialsProvider(provider);
} else {
clientBuilder.credentialsProvider(DefaultCredentialsProvider.create());
presignerBuilder.credentialsProvider(DefaultCredentialsProvider.create());
}
log.debug(
"Configured S3 {}: bucket={}, region={}, endpoint={}, pathStyle={}",
usage,
cfg.getBucket(),
region,
cfg.getEndpoint() == null || cfg.getEndpoint().isBlank()
? "<aws-default>"
: cfg.getEndpoint(),
cfg.isPathStyleAccess());
return new Bundle(clientBuilder.build(), presignerBuilder.build());
}
/**
* Block SSRF via the S3 endpoint setting. An admin who can edit config could otherwise point
* the SDK at the cloud metadata service (e.g. {@code http://169.254.169.254/}) and exfiltrate
* instance-role credentials. Reject any endpoint whose host resolves to a loopback, link-local,
* or RFC1918 private address unless the operator has explicitly opted in via {@code
* storage.s3.allow-private-endpoints=true}.
*/
static void validateEndpointHost(URI endpoint, boolean allowPrivate) {
if (allowPrivate) {
return;
}
String host = endpoint.getHost();
if (host == null || host.isBlank()) {
throw new IllegalStateException("storage.s3.endpoint must include a host: " + endpoint);
}
InetAddress[] addresses;
try {
addresses = InetAddress.getAllByName(host);
} catch (UnknownHostException e) {
throw new IllegalStateException(
"Unable to resolve storage.s3.endpoint host '" + host + "'", e);
}
for (InetAddress address : addresses) {
if (isPrivateOrLocal(address)) {
throw new IllegalStateException(
"storage.s3.endpoint host '"
+ host
+ "' resolves to private/link-local address "
+ address.getHostAddress()
+ "; set storage.s3.allow-private-endpoints=true to opt in"
+ " (e.g. for MinIO or in-cluster S3).");
}
}
}
private static boolean isPrivateOrLocal(InetAddress address) {
return address.isLoopbackAddress()
|| address.isLinkLocalAddress()
|| address.isSiteLocalAddress()
|| address.isAnyLocalAddress()
|| address.isMulticastAddress();
}
static RequestChecksumCalculation parseRequestChecksum(String value) {
if (value == null || value.isBlank()) {
return RequestChecksumCalculation.WHEN_SUPPORTED;
}
try {
return RequestChecksumCalculation.valueOf(
value.trim().toUpperCase(java.util.Locale.ROOT));
} catch (IllegalArgumentException ex) {
log.warn(
"Unknown storage.s3.request-checksum-calculation value '{}', falling back to WHEN_SUPPORTED",
value);
return RequestChecksumCalculation.WHEN_SUPPORTED;
}
}
static ResponseChecksumValidation parseResponseChecksum(String value) {
if (value == null || value.isBlank()) {
return ResponseChecksumValidation.WHEN_SUPPORTED;
}
try {
return ResponseChecksumValidation.valueOf(
value.trim().toUpperCase(java.util.Locale.ROOT));
} catch (IllegalArgumentException ex) {
log.warn(
"Unknown storage.s3.response-checksum-validation value '{}', falling back to WHEN_SUPPORTED",
value);
return ResponseChecksumValidation.WHEN_SUPPORTED;
}
}
}
@@ -0,0 +1,226 @@
package stirling.software.proprietary.cluster.s3;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.Optional;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.FileStore;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Exception;
/**
* S3-backed {@link FileStore} for transient job-result files. Objects are namespaced under a
* configurable key prefix (default {@code transient/}) and can coexist in the same bucket as {@code
* S3StorageProvider}.
*/
@Slf4j
public class S3FileStore implements FileStore, AutoCloseable {
public static final String DEFAULT_KEY_PREFIX = "transient/";
private final S3Client s3Client;
private final String bucket;
private final String keyPrefix;
private final boolean ownsClient;
public S3FileStore(S3Client s3Client, String bucket) {
this(s3Client, bucket, DEFAULT_KEY_PREFIX, true);
}
public S3FileStore(S3Client s3Client, String bucket, String keyPrefix) {
this(s3Client, bucket, keyPrefix, true);
}
/**
* @param ownsClient when true, {@link #close()} will close the supplied client. Set to false in
* tests that share the client with another consumer.
*/
public S3FileStore(S3Client s3Client, String bucket, String keyPrefix, boolean ownsClient) {
if (bucket == null || bucket.isBlank()) {
throw new IllegalArgumentException("S3 bucket must be configured");
}
this.s3Client = s3Client;
this.bucket = bucket;
this.keyPrefix = normalizePrefix(keyPrefix);
this.ownsClient = ownsClient;
}
@Override
public Stored store(InputStream in, String originalName) throws IOException {
String fileId = UUID.randomUUID().toString();
// S3 PUT requires a known content-length; spool to a temp file first so memory stays
// bounded for large payloads, then stream the file to S3 via RequestBody.fromFile.
Path tempFile = Files.createTempFile("s3-upload-", ".bin");
long size;
try {
try (InputStream src = in) {
Files.copy(src, tempFile, StandardCopyOption.REPLACE_EXISTING);
}
size = Files.size(tempFile);
PutObjectRequest request =
PutObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
try {
s3Client.putObject(request, RequestBody.fromFile(tempFile));
} catch (SdkException e) {
throw new IOException("Failed to upload object to S3", e);
}
} finally {
try {
Files.deleteIfExists(tempFile);
} catch (IOException cleanupError) {
log.warn("Failed to delete S3 upload temp file: {}", tempFile, cleanupError);
}
}
return new Stored(fileId, size);
}
@Override
public InputStream retrieve(String fileId) throws IOException {
validateFileId(fileId);
GetObjectRequest request =
GetObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
try {
ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(request);
return new BufferedInputStream(stream);
} catch (NoSuchKeyException e) {
throw new IOException("File not found with ID: " + fileId, e);
} catch (SdkException e) {
throw new IOException("Failed to load object from S3", e);
}
}
@Override
public byte[] retrieveBytes(String fileId) throws IOException {
validateFileId(fileId);
GetObjectRequest request =
GetObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
try (ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(request)) {
return stream.readAllBytes();
} catch (NoSuchKeyException e) {
throw new IOException("File not found with ID: " + fileId, e);
} catch (SdkException e) {
throw new IOException("Failed to load object from S3", e);
}
}
@Override
public long size(String fileId) throws IOException {
validateFileId(fileId);
HeadObjectRequest request =
HeadObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
try {
HeadObjectResponse response = s3Client.headObject(request);
return Optional.ofNullable(response.contentLength()).orElse(0L);
} catch (NoSuchKeyException e) {
throw new IOException("File not found with ID: " + fileId, e);
} catch (S3Exception e) {
if (e.statusCode() == 404) {
throw new IOException("File not found with ID: " + fileId, e);
}
throw new IOException("Failed to head object in S3", e);
} catch (SdkException e) {
throw new IOException("Failed to head object in S3", e);
}
}
@Override
public boolean delete(String fileId) {
try {
validateFileId(fileId);
} catch (IllegalArgumentException e) {
log.warn("Refusing to delete invalid file id: {}", fileId);
return false;
}
try {
s3Client.deleteObject(
DeleteObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build());
return true;
} catch (SdkException e) {
log.error("Error deleting file with ID: {}", fileId, e);
return false;
}
}
@Override
public boolean exists(String fileId) {
try {
validateFileId(fileId);
} catch (IllegalArgumentException e) {
return false;
}
HeadObjectRequest request =
HeadObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build();
try {
s3Client.headObject(request);
return true;
} catch (NoSuchKeyException e) {
return false;
} catch (S3Exception e) {
if (e.statusCode() == 404) {
return false;
}
log.warn("Error checking existence for file ID: {}", fileId, e);
return false;
} catch (SdkException e) {
log.warn("Error checking existence for file ID: {}", fileId, e);
return false;
}
}
@Override
public void close() {
if (!ownsClient) {
return;
}
try {
s3Client.close();
} catch (Exception e) {
log.warn("Error closing S3 client", e);
}
}
String resolveKey(String fileId) {
return keyPrefix + fileId;
}
private static void validateFileId(String fileId) {
if (fileId == null || fileId.isBlank()) {
throw new IllegalArgumentException("File ID must not be blank");
}
if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
throw new IllegalArgumentException("Invalid file ID");
}
}
private static String normalizePrefix(String prefix) {
if (prefix == null || prefix.isBlank()) {
return "";
}
String trimmed = prefix.trim();
if (trimmed.startsWith("/")) {
trimmed = trimmed.substring(1);
}
if (!trimmed.isEmpty() && !trimmed.endsWith("/")) {
trimmed = trimmed + "/";
}
return trimmed;
}
}
@@ -0,0 +1,37 @@
package stirling.software.proprietary.cluster.s3;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.cluster.FileStore;
import stirling.software.common.model.ApplicationProperties;
/** Activates the S3-backed transient {@link FileStore} when {@code cluster.artifactStore=s3}. */
@Slf4j
@Configuration
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "cluster", name = "artifactStore", havingValue = "s3")
public class S3FileStoreConfiguration {
private final ApplicationProperties applicationProperties;
@Bean(destroyMethod = "close")
@ConditionalOnMissingBean
public FileStore fileStore(@Value("${cluster.s3.keyPrefix:transient/}") String keyPrefix) {
ApplicationProperties.Storage.S3 cfg = applicationProperties.getStorage().getS3();
S3Clients.Bundle bundle = S3Clients.build(cfg, "cluster file store");
// FileStore has no signed-URL contract; close the unused presigner immediately.
try {
bundle.presigner().close();
} catch (Exception ignored) {
}
log.info("Cluster FileStore: s3 (bucket={}, keyPrefix={})", cfg.getBucket(), keyPrefix);
return new S3FileStore(bundle.client(), cfg.getBucket(), keyPrefix, true);
}
}
@@ -49,7 +49,11 @@ public class EEAppConfig {
@Profile("security & !saas")
@Bean(name = "SSOAutoLogin")
public boolean ssoAutoLogin() {
return applicationProperties.getPremium().getProFeatures().isSsoAutoLogin();
boolean enabled = applicationProperties.getPremium().getProFeatures().isSsoAutoLogin();
if (enabled) {
licenseKeyChecker.requireProOrEnterprise("premium.proFeatures.ssoAutoLogin=true");
}
return enabled;
}
// TODO: Remove post migration
@@ -32,7 +32,10 @@ public class LicenseKeyChecker {
private final UserLicenseSettingsService licenseSettingsService;
private License premiumEnabledResult = License.NORMAL;
// volatile: written by evaluateLicense() on the @Scheduled refresh thread, read by request
// threads via getPremiumLicenseEnabledResult() / requireProOrEnterprise(). Ensures readers see
// the latest tier rather than a stale cached value.
private volatile License premiumEnabledResult = License.NORMAL;
public LicenseKeyChecker(
KeygenLicenseVerifier licenseService,
@@ -133,4 +136,16 @@ public class LicenseKeyChecker {
public License getPremiumLicenseEnabledResult() {
return premiumEnabledResult;
}
/**
* Throws {@link IllegalStateException} if the current license is not Pro or Enterprise. Used by
* boot-time gates to fail fast when an operator enables a premium-only setting without a valid
* license. {@code configuredAs} is the human-readable property path (e.g. {@code
* "storage.provider=s3"}) and appears in the exception message.
*/
public void requireProOrEnterprise(String configuredAs) {
if (premiumEnabledResult != License.SERVER && premiumEnabledResult != License.ENTERPRISE) {
throw new IllegalStateException(configuredAs + " requires a Pro or Enterprise license");
}
}
}
@@ -0,0 +1,95 @@
package stirling.software.proprietary.storage.config;
import java.util.Locale;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.security.configuration.ee.LicenseKeyChecker;
/**
* Fails fast at boot if cluster mode is enabled with node-local storage. Validates both {@code
* storage.provider} (persistent uploads) and {@code cluster.artifactStore} (transient job-result
* files): neither may be {@code local} when {@code cluster.enabled=true}. Additionally enforces
* that any S3-backed configuration ({@code storage.provider=s3} or {@code
* cluster.artifactStore=s3}) is accompanied by a valid Pro / Enterprise license.
*/
@Configuration
@RequiredArgsConstructor
@Slf4j
public class ClusterStorageGate {
private final ApplicationProperties applicationProperties;
private final LicenseKeyChecker licenseKeyChecker;
@Value("${cluster.enabled:false}")
private boolean clusterEnabled;
@Value("${cluster.artifactStore:local}")
private String clusterArtifactStore;
@PostConstruct
void validate() {
// License enforcement runs regardless of cluster.enabled: even a single-node setup that
// selects a remote backend must hold a Pro or higher license.
ApplicationProperties.Storage storage = applicationProperties.getStorage();
if (storage != null && storage.isEnabled()) {
String provider = normalize(storage.getProvider());
if ("s3".equals(provider) || "database".equals(provider)) {
licenseKeyChecker.requireProOrEnterprise("storage.provider=" + provider);
}
}
if ("s3".equals(normalize(clusterArtifactStore))) {
licenseKeyChecker.requireProOrEnterprise("cluster.artifactStore=s3");
}
if (!clusterEnabled) {
return;
}
if (storage != null && storage.isEnabled()) {
validate(
"storage.provider",
storage.getProvider(),
"Local filesystem storage cannot be shared across cluster nodes."
+ " Configure storage.provider=s3 (with storage.s3.bucket /"
+ " endpoint / credentials) or storage.provider=database before"
+ " enabling clustering.");
}
validate(
"cluster.artifactStore",
clusterArtifactStore,
"Per-node disk cannot back transient job-result files in a multi-node"
+ " deployment; downloads would 404 whenever the load balancer routes"
+ " a follow-up request to a different node. Configure"
+ " cluster.artifactStore=s3 (reuses storage.s3.* config)"
+ " before enabling clustering.");
}
private static String normalize(String value) {
return Optional.ofNullable(value).orElse("local").trim().toLowerCase(Locale.ROOT);
}
private static void validate(String propertyName, String configuredValue, String remediation) {
String normalized =
Optional.ofNullable(configuredValue)
.orElse("local")
.trim()
.toLowerCase(Locale.ROOT);
if ("local".equals(normalized)) {
throw new IllegalStateException(
"Cluster mode (cluster.enabled=true) is incompatible with "
+ propertyName
+ "=local. "
+ remediation);
}
log.info(
"Cluster storage gate: clusterEnabled=true, {}={} -> OK", propertyName, normalized);
}
}
@@ -15,8 +15,11 @@ import lombok.extern.slf4j.Slf4j;
import stirling.software.common.configuration.InstallationPathConfig;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.cluster.s3.S3Clients;
import stirling.software.proprietary.security.configuration.ee.LicenseKeyChecker;
import stirling.software.proprietary.storage.provider.DatabaseStorageProvider;
import stirling.software.proprietary.storage.provider.LocalStorageProvider;
import stirling.software.proprietary.storage.provider.S3StorageProvider;
import stirling.software.proprietary.storage.provider.StorageProvider;
import stirling.software.proprietary.storage.repository.StoredFileBlobRepository;
@@ -27,8 +30,9 @@ public class StorageProviderConfig {
private final ApplicationProperties applicationProperties;
private final StoredFileBlobRepository storedFileBlobRepository;
private final LicenseKeyChecker licenseKeyChecker;
@Bean
@Bean(destroyMethod = "close")
public StorageProvider storageProvider() {
boolean storageEnabled = applicationProperties.getStorage().isEnabled();
String providerName =
@@ -37,8 +41,13 @@ public class StorageProviderConfig {
.trim()
.toLowerCase(Locale.ROOT);
if ("database".equals(providerName)) {
licenseKeyChecker.requireProOrEnterprise("storage.provider=database");
return new DatabaseStorageProvider(storedFileBlobRepository);
}
if ("s3".equals(providerName)) {
licenseKeyChecker.requireProOrEnterprise("storage.provider=s3");
return buildS3Provider(applicationProperties.getStorage().getS3());
}
if (!"local".equals(providerName)) {
throw new IllegalStateException("Storage provider not supported: " + providerName);
}
@@ -71,4 +80,9 @@ public class StorageProviderConfig {
}
return new LocalStorageProvider(basePath);
}
private S3StorageProvider buildS3Provider(ApplicationProperties.Storage.S3 cfg) {
S3Clients.Bundle bundle = S3Clients.build(cfg, "storage provider");
return new S3StorageProvider(bundle.client(), bundle.presigner(), cfg.getBucket());
}
}
@@ -1,7 +1,11 @@
package stirling.software.proprietary.storage.controller;
import java.io.IOException;
import java.net.URI;
import java.time.Duration;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import org.springframework.http.ContentDisposition;
import org.springframework.http.HttpHeaders;
@@ -25,6 +29,7 @@ import org.springframework.web.server.ResponseStatusException;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.proprietary.security.model.User;
import stirling.software.proprietary.storage.model.FileShare;
@@ -35,17 +40,22 @@ import stirling.software.proprietary.storage.model.api.ShareLinkMetadataResponse
import stirling.software.proprietary.storage.model.api.ShareLinkResponse;
import stirling.software.proprietary.storage.model.api.ShareWithUserRequest;
import stirling.software.proprietary.storage.model.api.StoredFileResponse;
import stirling.software.proprietary.storage.provider.StorageProvider;
import stirling.software.proprietary.storage.service.FileStorageService;
@RestController
@RequestMapping("/api/v1/storage")
@RequiredArgsConstructor
@Slf4j
@Tag(
name = "File Storage",
description = "Stored file management, sharing, and share link operations")
public class FileStorageController {
private static final Duration SIGNED_URL_TTL = Duration.ofMinutes(5);
private final FileStorageService fileStorageService;
private final StorageProvider storageProvider;
@PostMapping(
value = "/files",
@@ -91,7 +101,9 @@ public class FileStorageController {
User user = fileStorageService.requireAuthenticatedUser();
StoredFile file = fileStorageService.getAccessibleFile(user, fileId);
fileStorageService.requireReadAccess(user, file);
return buildFileResponse(file, inline);
Optional<ResponseEntity<org.springframework.core.io.Resource>> redirect =
tryRedirectToSignedUrl(file, inline);
return redirect.orElseGet(() -> buildFileResponse(file, inline));
}
@DeleteMapping("/files/{fileId}")
@@ -189,7 +201,9 @@ public class FileStorageController {
fileStorageService.requireReadAccess(share);
fileStorageService.recordShareAccess(share, authentication, inline);
StoredFile file = share.getFile();
return buildFileResponse(file, inline);
Optional<ResponseEntity<org.springframework.core.io.Resource>> redirect =
tryRedirectToSignedUrl(file, inline);
return redirect.orElseGet(() -> buildFileResponse(file, inline));
}
@GetMapping("/share-links/{token}/metadata")
@@ -272,4 +286,34 @@ public class FileStorageController {
&& authentication.isAuthenticated()
&& !"anonymousUser".equals(authentication.getPrincipal());
}
private Optional<ResponseEntity<org.springframework.core.io.Resource>> tryRedirectToSignedUrl(
StoredFile file, boolean inline) {
if (file == null || file.getStorageKey() == null || file.getStorageKey().isBlank()) {
return Optional.empty();
}
try {
Optional<URI> signed =
storageProvider.signedDownloadUrl(
file.getStorageKey(),
SIGNED_URL_TTL,
inline,
file.getOriginalFilename());
if (signed.isEmpty()) {
return Optional.empty();
}
HttpHeaders headers = new HttpHeaders();
headers.setLocation(signed.get());
ResponseEntity<org.springframework.core.io.Resource> response =
ResponseEntity.status(HttpStatus.FOUND).headers(headers).build();
return Optional.of(response);
} catch (IOException e) {
log.warn(
"Failed to create signed download URL for file {} (key: {}), falling back to streaming",
file.getId(),
file.getStorageKey(),
e);
return Optional.empty();
}
}
}
@@ -24,6 +24,9 @@ public class LocalStorageProvider implements StorageProvider {
@Override
public StoredObject store(User owner, MultipartFile file) throws IOException {
if (owner == null || owner.getId() == null) {
throw new IllegalArgumentException("owner.id is required for local storage key");
}
String originalFilename = sanitizeFilename(file.getOriginalFilename());
String storageKey =
owner.getId()
@@ -77,6 +80,7 @@ public class LocalStorageProvider implements StorageProvider {
if (filename == null || filename.isBlank()) {
return "file";
}
return Paths.get(filename).getFileName().toString();
String stripped = Paths.get(filename).getFileName().toString().replaceAll("\\p{Cntrl}", "");
return stripped.isBlank() ? "file" : stripped;
}
}
@@ -0,0 +1,190 @@
package stirling.software.proprietary.storage.provider;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.Optional;
import java.util.UUID;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.web.multipart.MultipartFile;
import lombok.extern.slf4j.Slf4j;
import stirling.software.proprietary.security.model.User;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
/** {@link StorageProvider} backed by an S3-compatible object store. */
@Slf4j
public class S3StorageProvider implements StorageProvider, AutoCloseable {
private final S3Client s3Client;
private final S3Presigner s3Presigner;
private final String bucket;
public S3StorageProvider(S3Client s3Client, S3Presigner s3Presigner, String bucket) {
if (bucket == null || bucket.isBlank()) {
throw new IllegalArgumentException("S3 bucket must be configured");
}
this.s3Client = s3Client;
this.s3Presigner = s3Presigner;
this.bucket = bucket;
}
@Override
public StoredObject store(User owner, MultipartFile file) throws IOException {
if (owner == null || owner.getId() == null) {
throw new IllegalArgumentException("owner.id is required for S3 storage key");
}
String originalFilename = sanitizeFilename(file.getOriginalFilename());
// Key is opaque ({ownerId}/{uuid}) so non-ASCII filenames don't break vendors that
// restrict key charset (e.g. Supabase Storage returns 400 Invalid key on unicode).
// The display name is preserved in StoredObject.originalFilename and the DB row.
String storageKey = owner.getId() + "/" + UUID.randomUUID();
PutObjectRequest.Builder request =
PutObjectRequest.builder().bucket(bucket).key(storageKey);
if (file.getContentType() != null && !file.getContentType().isBlank()) {
request.contentType(file.getContentType());
}
try (InputStream inputStream = file.getInputStream()) {
s3Client.putObject(
request.build(), RequestBody.fromInputStream(inputStream, file.getSize()));
} catch (SdkException e) {
throw new IOException("Failed to upload object to S3", e);
}
return StoredObject.builder()
.storageKey(storageKey)
.originalFilename(originalFilename)
.contentType(file.getContentType())
.sizeBytes(file.getSize())
.build();
}
@Override
public Resource load(String storageKey) throws IOException {
GetObjectRequest request =
GetObjectRequest.builder().bucket(bucket).key(storageKey).build();
try {
ResponseInputStream<GetObjectResponse> stream = s3Client.getObject(request);
long contentLength =
stream.response().contentLength() != null
? stream.response().contentLength()
: -1;
return new InputStreamResource(stream) {
@Override
public long contentLength() {
return contentLength;
}
};
} catch (NoSuchKeyException e) {
throw new IOException("File not found", e);
} catch (SdkException e) {
throw new IOException("Failed to load object from S3", e);
}
}
@Override
public void delete(String storageKey) throws IOException {
try {
s3Client.deleteObject(
DeleteObjectRequest.builder().bucket(bucket).key(storageKey).build());
} catch (SdkException e) {
throw new IOException("Failed to delete object from S3", e);
}
}
@Override
public Optional<URI> signedDownloadUrl(String storageKey, Duration ttl) throws IOException {
return signedDownloadUrl(storageKey, ttl, false, null);
}
@Override
public Optional<URI> signedDownloadUrl(
String storageKey, Duration ttl, boolean inline, String originalFilename)
throws IOException {
if (storageKey == null || storageKey.isBlank()) {
return Optional.empty();
}
Duration effectiveTtl =
ttl == null || ttl.isZero() || ttl.isNegative() ? Duration.ofMinutes(5) : ttl;
try {
GetObjectRequest.Builder getBuilder =
GetObjectRequest.builder().bucket(bucket).key(storageKey);
String disposition = buildContentDisposition(inline, originalFilename);
if (disposition != null) {
getBuilder.responseContentDisposition(disposition);
}
GetObjectPresignRequest presignRequest =
GetObjectPresignRequest.builder()
.signatureDuration(effectiveTtl)
.getObjectRequest(getBuilder.build())
.build();
PresignedGetObjectRequest presigned = s3Presigner.presignGetObject(presignRequest);
return Optional.of(presigned.url().toURI());
} catch (SdkException | URISyntaxException e) {
log.warn("Failed to create presigned S3 GET URL for key {}", storageKey, e);
return Optional.empty();
}
}
// Returns null when originalFilename is blank; S3 falls back to its own default in that case.
static String buildContentDisposition(boolean inline, String originalFilename) {
if (originalFilename == null || originalFilename.isBlank()) {
return null;
}
// Strip CR/LF and other control chars before path parsing (Paths.get throws on them on
// Windows, and they defeat header parsers).
String stripped = originalFilename.replaceAll("\\p{Cntrl}", "");
// Use only the basename to avoid leaking directory structure into the header.
int lastSeparator = Math.max(stripped.lastIndexOf('/'), stripped.lastIndexOf('\\'));
if (lastSeparator >= 0) {
stripped = stripped.substring(lastSeparator + 1);
}
if (stripped.isBlank()) {
return null;
}
// Escape per RFC 6266 quoted-string rules.
String escaped = stripped.replace("\\", "\\\\").replace("\"", "\\\"");
return (inline ? "inline" : "attachment") + "; filename=\"" + escaped + "\"";
}
@Override
public void close() {
try {
s3Presigner.close();
} catch (Exception e) {
log.warn("Error closing S3 presigner", e);
}
try {
s3Client.close();
} catch (Exception e) {
log.warn("Error closing S3 client", e);
}
}
private String sanitizeFilename(String filename) {
if (filename == null || filename.isBlank()) {
return "file";
}
String stripped = Paths.get(filename).getFileName().toString().replaceAll("\\p{Cntrl}", "");
return stripped.isBlank() ? "file" : stripped;
}
}
@@ -1,16 +1,45 @@
package stirling.software.proprietary.storage.provider;
import java.io.IOException;
import java.net.URI;
import java.time.Duration;
import java.util.Optional;
import org.springframework.core.io.Resource;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.proprietary.security.model.User;
public interface StorageProvider {
public interface StorageProvider extends AutoCloseable {
StoredObject store(User owner, MultipartFile file) throws IOException;
Resource load(String storageKey) throws IOException;
void delete(String storageKey) throws IOException;
/**
* Releases any backend-specific resources. Default no-op so {@link LocalStorageProvider} and
* {@link DatabaseStorageProvider} (which hold no closeable handles) satisfy Spring's
* {@code @Bean(destroyMethod = "close")} signature requirement without ceremony. {@code
* S3StorageProvider} overrides this to close the underlying SDK client + presigner.
*/
@Override
default void close() {}
/**
* Returns a presigned download URL valid for {@code ttl}, or {@link Optional#empty()} if the
* provider does not support signed URLs (callers fall back to {@link #load(String)}).
*/
default Optional<URI> signedDownloadUrl(String storageKey, Duration ttl) throws IOException {
return signedDownloadUrl(storageKey, ttl, false, null);
}
/**
* Like {@link #signedDownloadUrl(String, Duration)} with explicit Content-Disposition control.
*/
default Optional<URI> signedDownloadUrl(
String storageKey, Duration ttl, boolean inline, String originalFilename)
throws IOException {
return Optional.empty();
}
}