From b3c4b8b463b92607acfb065bd9d02575b1cdeac6 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Thu, 28 May 2026 13:06:27 +0100 Subject: [PATCH] Add S3 storage and cluster artifact backend (#6457) --- app/allowed-licenses.json | 4 + .../common/model/ApplicationProperties.java | 52 ++ .../src/main/resources/settings.yml.template | 35 + app/proprietary/build.gradle | 9 + .../proprietary/cluster/s3/S3Clients.java | 200 +++++ .../proprietary/cluster/s3/S3FileStore.java | 226 +++++ .../cluster/s3/S3FileStoreConfiguration.java | 37 + .../configuration/ee/EEAppConfig.java | 6 +- .../configuration/ee/LicenseKeyChecker.java | 17 +- .../storage/config/ClusterStorageGate.java | 95 +++ .../storage/config/StorageProviderConfig.java | 16 +- .../controller/FileStorageController.java | 48 +- .../provider/LocalStorageProvider.java | 6 +- .../storage/provider/S3StorageProvider.java | 190 +++++ .../storage/provider/StorageProvider.java | 31 +- .../proprietary/cluster/s3/S3ClientsTest.java | 147 ++++ .../cluster/s3/S3FileStoreTest.java | 253 ++++++ .../cluster/s3/S3VendorComprehensiveTest.java | 779 ++++++++++++++++++ .../cluster/s3/S3VendorSmokeTest.java | 161 ++++ .../configuration/ee/EEAppConfigTest.java | 59 ++ .../ee/LicenseKeyCheckerTest.java | 41 + .../config/ClusterStorageGateTest.java | 250 ++++++ .../config/StorageProviderConfigTest.java | 116 +++ .../controller/FileStorageControllerTest.java | 130 +++ .../provider/S3StorageProviderTest.java | 282 +++++++ 25 files changed, 3183 insertions(+), 7 deletions(-) create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3Clients.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3FileStore.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3FileStoreConfiguration.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/storage/config/ClusterStorageGate.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/S3StorageProvider.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3ClientsTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3FileStoreTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3VendorComprehensiveTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3VendorSmokeTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/security/configuration/ee/EEAppConfigTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/storage/config/ClusterStorageGateTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/storage/config/StorageProviderConfigTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/storage/controller/FileStorageControllerTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/storage/provider/S3StorageProviderTest.java diff --git a/app/allowed-licenses.json b/app/allowed-licenses.json index 315c6bb18..cd2fe06a3 100644 --- a/app/allowed-licenses.json +++ b/app/allowed-licenses.json @@ -44,6 +44,10 @@ "moduleName": ".*", "moduleLicense": "The MIT License" }, + { + "moduleName": ".*", + "moduleLicense": "MIT-0" + }, { "moduleName": "com.github.jai-imageio:jai-imageio-core", "moduleLicense": "LICENSE.txt" diff --git a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java index cf3a9d201..bb2eb8167 100644 --- a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java +++ b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java @@ -788,6 +788,7 @@ public class ApplicationProperties { private boolean enabled = false; private String provider = "local"; private Local local = new Local(); + private S3 s3 = new S3(); private Quotas quotas = new Quotas(); private Sharing sharing = new Sharing(); private Signing signing = new Signing(); @@ -797,6 +798,57 @@ public class ApplicationProperties { private String basePath = InstallationPathConfig.getPath() + "storage"; } + @Data + public static class S3 { + /** + * Optional custom endpoint (e.g. {@code https://.r2.cloudflarestorage.com}, + * {@code https://.supabase.co/storage/v1/s3}, or {@code http://localhost:9000} + * for MinIO). Blank = use AWS regional default. + */ + private String endpoint = ""; + + private String bucket = ""; + + private String region = "us-east-1"; + + private String accessKey = ""; + private String secretKey = ""; + + /** + * When {@code true} use path-style URLs ({@code //}) instead of + * virtual-hosted ({@code ./}). MinIO and most S3-compatible + * gateways require path-style; AWS S3 prefers virtual-hosted. + */ + private boolean pathStyleAccess = false; + + /** + * When {@code false} (default), {@code endpoint} hostnames that resolve to private, + * loopback, or link-local addresses are rejected at startup to block SSRF attacks via + * the cloud metadata service (e.g. {@code http://169.254.169.254/}). Set to {@code + * true} to opt in for MinIO / in-cluster S3 endpoints on private networks. + */ + private boolean allowPrivateEndpoints = false; + + /** + * Controls when the SDK adds an {@code x-amz-checksum-*} header on PUT/UploadPart. + * Default {@code WHEN_SUPPORTED} (the SDK default since 2.30) makes the SDK send a + * CRC32 checksum on every upload - this works on AWS S3, MinIO, current Supabase, + * Backblaze B2 (post-July-2025), and modern R2. Set to {@code WHEN_REQUIRED} to + * suppress the auto-checksum on vendors that reject unknown {@code x-amz-checksum-*} + * headers (older Backblaze B2, some R2 corner cases, GCS S3 endpoint). Invalid values + * fall back to {@code WHEN_SUPPORTED}. + */ + private String requestChecksumCalculation = "WHEN_SUPPORTED"; + + /** + * Controls when the SDK validates returned {@code x-amz-checksum-*} headers on GET + * responses. Default {@code WHEN_SUPPORTED}. Set to {@code WHEN_REQUIRED} if your + * vendor never returns these headers and you see false-positive checksum-mismatch + * errors. Invalid values fall back to {@code WHEN_SUPPORTED}. + */ + private String responseChecksumValidation = "WHEN_SUPPORTED"; + } + @Data public static class Sharing { private boolean enabled = false; diff --git a/app/core/src/main/resources/settings.yml.template b/app/core/src/main/resources/settings.yml.template index 7f686d582..a82f4d0f9 100644 --- a/app/core/src/main/resources/settings.yml.template +++ b/app/core/src/main/resources/settings.yml.template @@ -246,6 +246,39 @@ storage: provider: local # storage provider: 'local' for filesystem storage, 'database' for DB-backed storage local: basePath: './storage' # base directory for stored files + # ==================================================================================== + # S3-COMPATIBLE OBJECT STORAGE - PRO / ENTERPRISE LICENSE REQUIRED + # storage.provider=s3, storage.provider=database, and cluster.artifactStore=s3 all + # require a valid Pro or Enterprise license. + # ==================================================================================== + # Used when provider=s3 (persistent user uploads) and/or cluster.artifactStore=s3 + # (transient cluster artifacts). The two consumers share this block. + # Vendor cheat sheet (set the highlighted flags to taste): + # AWS S3 -> endpoint='' region='' pathStyleAccess=false + # Cloudflare R2 -> endpoint='https://.r2.cloudflarestorage.com' region='auto' + # pathStyleAccess=false; if uploads fail with 'unsupported header + # x-amz-checksum-*' set requestChecksumCalculation=WHEN_REQUIRED + # Supabase Storage -> endpoint='https://.supabase.co/storage/v1/s3' + # region='' pathStyleAccess=true + # (filenames with non-ASCII display fine - the storage key is opaque) + # MinIO (in-cluster) -> endpoint='http://minio:9000' region='us-east-1' + # pathStyleAccess=true allowPrivateEndpoints=true + # Backblaze B2 -> endpoint='https://s3..backblazeb2.com' + # If on a B2 deployment older than July-2025 and uploads return + # 'Unsupported header x-amz-checksum-crc32', set + # requestChecksumCalculation=WHEN_REQUIRED + # DigitalOcean Spaces -> endpoint='https://.digitaloceanspaces.com' + # Note: 5GB per-object cap (regardless of multipart) + s3: + endpoint: "" # blank = use AWS regional default; otherwise full URL incl. https:// + bucket: "" # required when provider=s3 or cluster.artifactStore=s3 + region: us-east-1 + accessKey: "" # blank = fall back to AWS DefaultCredentialsProvider (env / profile / IMDS) + secretKey: "" + pathStyleAccess: false # true for MinIO and Supabase; false for AWS/R2/most CDNs + allowPrivateEndpoints: false # true required when endpoint resolves to a private/loopback IP (e.g. in-cluster MinIO). SSRF guard - leave false for any internet-facing vendor. + requestChecksumCalculation: WHEN_SUPPORTED # WHEN_SUPPORTED|WHEN_REQUIRED|DISABLED. Set WHEN_REQUIRED if your vendor rejects auto-added x-amz-checksum-* headers (older Backblaze B2, some R2 corner cases). + responseChecksumValidation: WHEN_SUPPORTED # WHEN_SUPPORTED|WHEN_REQUIRED|DISABLED. Set WHEN_REQUIRED if you see false-positive checksum-mismatch errors on GET from a vendor that never returns checksum headers. quotas: maxStorageMbPerUser: -1 # Max storage per user in MB; -1 disables per-user cap maxStorageMbTotal: -1 # Max storage across all users in MB; -1 disables total cap @@ -336,6 +369,8 @@ cluster: enabled: false # Master switch. 'false' (default) wires the in-process backplane and skips all cluster checks. Single-instance installs do not need to change anything here. backplane: inprocess # Backplane implementation: 'inprocess' (single JVM only) or 'valkey' (multi-node via Valkey/Redis) artifactStore: local # Transient cluster job-artifact backend: 'local' (per-node disk; single-node only) or 's3' (shared object store; required for multi-node). Distinct from 'storage.provider' which controls persistent user uploads - when both are 's3' they share the storage.s3.* credentials block. Multi-node deployments MUST set this to 's3'. + s3: + keyPrefix: transient/ # Bucket key prefix used by the cluster artifact store when artifactStore=s3. Trailing slash recommended. Lets a single bucket host both persistent uploads (storage.s3.*) and transient job artifacts under separate prefixes. valkey: url: "" # Valkey/Redis URL, e.g. 'redis://valkey:6379' or 'rediss://...' for TLS. Required when enabled=true and backplane=valkey. tls: diff --git a/app/proprietary/build.gradle b/app/proprietary/build.gradle index e923ae622..4c9932d99 100644 --- a/app/proprietary/build.gradle +++ b/app/proprietary/build.gradle @@ -5,6 +5,8 @@ repositories { ext { jwtVersion = '0.13.0' + awsSdkVersion = '2.44.12' + testcontainersMinioVersion = '1.21.4' } bootRun { @@ -71,6 +73,13 @@ dependencies { implementation('com.coveo:saml-client:5.0.0') { exclude group: 'org.opensaml', module: 'opensaml-core' } + + implementation "software.amazon.awssdk:s3:$awsSdkVersion" + implementation "software.amazon.awssdk:url-connection-client:$awsSdkVersion" + + testImplementation "org.testcontainers:minio:$testcontainersMinioVersion" + testImplementation "org.testcontainers:junit-jupiter:$testcontainersMinioVersion" + testImplementation "org.testcontainers:localstack:$testcontainersMinioVersion" } tasks.register('prepareKotlinBuildScriptModel') {} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3Clients.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3Clients.java new file mode 100644 index 000000000..eacdaf32c --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3Clients.java @@ -0,0 +1,200 @@ +package stirling.software.proprietary.cluster.s3; + +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.ApplicationProperties; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; +import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; + +/** + * Shared factory for {@link S3Client} and {@link S3Presigner} instances used by both {@code + * S3StorageProvider} and {@code S3FileStore}, so endpoint/region/credentials wiring lives in + * exactly one place. + */ +@Slf4j +public final class S3Clients { + + private S3Clients() {} + + /** Paired client and presigner with coordinated lifecycle. */ + public record Bundle(S3Client client, S3Presigner presigner) implements AutoCloseable { + @Override + public void close() { + try { + presigner.close(); + } catch (Exception e) { + log.warn("Error closing S3 presigner", e); + } + try { + client.close(); + } catch (Exception e) { + log.warn("Error closing S3 client", e); + } + } + } + + /** Build a client+presigner pair from the shared S3 config block. */ + public static Bundle build(ApplicationProperties.Storage.S3 cfg, String usage) { + if (cfg == null) { + throw new IllegalStateException( + usage + " requires storage.s3.* configuration to be set"); + } + if (cfg.getBucket() == null || cfg.getBucket().isBlank()) { + throw new IllegalStateException(usage + " requires storage.s3.bucket to be set"); + } + String region = + cfg.getRegion() == null || cfg.getRegion().isBlank() + ? "us-east-1" + : cfg.getRegion(); + + S3Configuration s3Configuration = + S3Configuration.builder().pathStyleAccessEnabled(cfg.isPathStyleAccess()).build(); + + RequestChecksumCalculation requestChecksum = + parseRequestChecksum(cfg.getRequestChecksumCalculation()); + ResponseChecksumValidation responseChecksum = + parseResponseChecksum(cfg.getResponseChecksumValidation()); + + S3ClientBuilder clientBuilder = + S3Client.builder() + .httpClient(UrlConnectionHttpClient.create()) + .region(Region.of(region)) + .serviceConfiguration(s3Configuration) + .requestChecksumCalculation(requestChecksum) + .responseChecksumValidation(responseChecksum); + + S3Presigner.Builder presignerBuilder = + S3Presigner.builder() + .region(Region.of(region)) + .serviceConfiguration(s3Configuration); + + if (cfg.getEndpoint() != null && !cfg.getEndpoint().isBlank()) { + URI endpoint; + try { + endpoint = new URI(cfg.getEndpoint()); + } catch (URISyntaxException e) { + throw new IllegalStateException( + "Invalid storage.s3.endpoint: " + cfg.getEndpoint(), e); + } + validateEndpointHost(endpoint, cfg.isAllowPrivateEndpoints()); + clientBuilder.endpointOverride(endpoint); + presignerBuilder.endpointOverride(endpoint); + } + + boolean hasStaticCreds = + cfg.getAccessKey() != null + && !cfg.getAccessKey().isBlank() + && cfg.getSecretKey() != null + && !cfg.getSecretKey().isBlank(); + if (hasStaticCreds) { + AwsBasicCredentials credentials = + AwsBasicCredentials.create(cfg.getAccessKey(), cfg.getSecretKey()); + StaticCredentialsProvider provider = StaticCredentialsProvider.create(credentials); + clientBuilder.credentialsProvider(provider); + presignerBuilder.credentialsProvider(provider); + } else { + clientBuilder.credentialsProvider(DefaultCredentialsProvider.create()); + presignerBuilder.credentialsProvider(DefaultCredentialsProvider.create()); + } + + log.debug( + "Configured S3 {}: bucket={}, region={}, endpoint={}, pathStyle={}", + usage, + cfg.getBucket(), + region, + cfg.getEndpoint() == null || cfg.getEndpoint().isBlank() + ? "" + : cfg.getEndpoint(), + cfg.isPathStyleAccess()); + + return new Bundle(clientBuilder.build(), presignerBuilder.build()); + } + + /** + * Block SSRF via the S3 endpoint setting. An admin who can edit config could otherwise point + * the SDK at the cloud metadata service (e.g. {@code http://169.254.169.254/}) and exfiltrate + * instance-role credentials. Reject any endpoint whose host resolves to a loopback, link-local, + * or RFC1918 private address unless the operator has explicitly opted in via {@code + * storage.s3.allow-private-endpoints=true}. + */ + static void validateEndpointHost(URI endpoint, boolean allowPrivate) { + if (allowPrivate) { + return; + } + String host = endpoint.getHost(); + if (host == null || host.isBlank()) { + throw new IllegalStateException("storage.s3.endpoint must include a host: " + endpoint); + } + InetAddress[] addresses; + try { + addresses = InetAddress.getAllByName(host); + } catch (UnknownHostException e) { + throw new IllegalStateException( + "Unable to resolve storage.s3.endpoint host '" + host + "'", e); + } + for (InetAddress address : addresses) { + if (isPrivateOrLocal(address)) { + throw new IllegalStateException( + "storage.s3.endpoint host '" + + host + + "' resolves to private/link-local address " + + address.getHostAddress() + + "; set storage.s3.allow-private-endpoints=true to opt in" + + " (e.g. for MinIO or in-cluster S3)."); + } + } + } + + private static boolean isPrivateOrLocal(InetAddress address) { + return address.isLoopbackAddress() + || address.isLinkLocalAddress() + || address.isSiteLocalAddress() + || address.isAnyLocalAddress() + || address.isMulticastAddress(); + } + + static RequestChecksumCalculation parseRequestChecksum(String value) { + if (value == null || value.isBlank()) { + return RequestChecksumCalculation.WHEN_SUPPORTED; + } + try { + return RequestChecksumCalculation.valueOf( + value.trim().toUpperCase(java.util.Locale.ROOT)); + } catch (IllegalArgumentException ex) { + log.warn( + "Unknown storage.s3.request-checksum-calculation value '{}', falling back to WHEN_SUPPORTED", + value); + return RequestChecksumCalculation.WHEN_SUPPORTED; + } + } + + static ResponseChecksumValidation parseResponseChecksum(String value) { + if (value == null || value.isBlank()) { + return ResponseChecksumValidation.WHEN_SUPPORTED; + } + try { + return ResponseChecksumValidation.valueOf( + value.trim().toUpperCase(java.util.Locale.ROOT)); + } catch (IllegalArgumentException ex) { + log.warn( + "Unknown storage.s3.response-checksum-validation value '{}', falling back to WHEN_SUPPORTED", + value); + return ResponseChecksumValidation.WHEN_SUPPORTED; + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3FileStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3FileStore.java new file mode 100644 index 000000000..1cff20e6a --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3FileStore.java @@ -0,0 +1,226 @@ +package stirling.software.proprietary.cluster.s3; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Optional; +import java.util.UUID; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.cluster.FileStore; + +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; + +/** + * S3-backed {@link FileStore} for transient job-result files. Objects are namespaced under a + * configurable key prefix (default {@code transient/}) and can coexist in the same bucket as {@code + * S3StorageProvider}. + */ +@Slf4j +public class S3FileStore implements FileStore, AutoCloseable { + + public static final String DEFAULT_KEY_PREFIX = "transient/"; + + private final S3Client s3Client; + private final String bucket; + private final String keyPrefix; + private final boolean ownsClient; + + public S3FileStore(S3Client s3Client, String bucket) { + this(s3Client, bucket, DEFAULT_KEY_PREFIX, true); + } + + public S3FileStore(S3Client s3Client, String bucket, String keyPrefix) { + this(s3Client, bucket, keyPrefix, true); + } + + /** + * @param ownsClient when true, {@link #close()} will close the supplied client. Set to false in + * tests that share the client with another consumer. + */ + public S3FileStore(S3Client s3Client, String bucket, String keyPrefix, boolean ownsClient) { + if (bucket == null || bucket.isBlank()) { + throw new IllegalArgumentException("S3 bucket must be configured"); + } + this.s3Client = s3Client; + this.bucket = bucket; + this.keyPrefix = normalizePrefix(keyPrefix); + this.ownsClient = ownsClient; + } + + @Override + public Stored store(InputStream in, String originalName) throws IOException { + String fileId = UUID.randomUUID().toString(); + // S3 PUT requires a known content-length; spool to a temp file first so memory stays + // bounded for large payloads, then stream the file to S3 via RequestBody.fromFile. + Path tempFile = Files.createTempFile("s3-upload-", ".bin"); + long size; + try { + try (InputStream src = in) { + Files.copy(src, tempFile, StandardCopyOption.REPLACE_EXISTING); + } + size = Files.size(tempFile); + PutObjectRequest request = + PutObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build(); + try { + s3Client.putObject(request, RequestBody.fromFile(tempFile)); + } catch (SdkException e) { + throw new IOException("Failed to upload object to S3", e); + } + } finally { + try { + Files.deleteIfExists(tempFile); + } catch (IOException cleanupError) { + log.warn("Failed to delete S3 upload temp file: {}", tempFile, cleanupError); + } + } + return new Stored(fileId, size); + } + + @Override + public InputStream retrieve(String fileId) throws IOException { + validateFileId(fileId); + GetObjectRequest request = + GetObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build(); + try { + ResponseInputStream stream = s3Client.getObject(request); + return new BufferedInputStream(stream); + } catch (NoSuchKeyException e) { + throw new IOException("File not found with ID: " + fileId, e); + } catch (SdkException e) { + throw new IOException("Failed to load object from S3", e); + } + } + + @Override + public byte[] retrieveBytes(String fileId) throws IOException { + validateFileId(fileId); + GetObjectRequest request = + GetObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build(); + try (ResponseInputStream stream = s3Client.getObject(request)) { + return stream.readAllBytes(); + } catch (NoSuchKeyException e) { + throw new IOException("File not found with ID: " + fileId, e); + } catch (SdkException e) { + throw new IOException("Failed to load object from S3", e); + } + } + + @Override + public long size(String fileId) throws IOException { + validateFileId(fileId); + HeadObjectRequest request = + HeadObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build(); + try { + HeadObjectResponse response = s3Client.headObject(request); + return Optional.ofNullable(response.contentLength()).orElse(0L); + } catch (NoSuchKeyException e) { + throw new IOException("File not found with ID: " + fileId, e); + } catch (S3Exception e) { + if (e.statusCode() == 404) { + throw new IOException("File not found with ID: " + fileId, e); + } + throw new IOException("Failed to head object in S3", e); + } catch (SdkException e) { + throw new IOException("Failed to head object in S3", e); + } + } + + @Override + public boolean delete(String fileId) { + try { + validateFileId(fileId); + } catch (IllegalArgumentException e) { + log.warn("Refusing to delete invalid file id: {}", fileId); + return false; + } + try { + s3Client.deleteObject( + DeleteObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build()); + return true; + } catch (SdkException e) { + log.error("Error deleting file with ID: {}", fileId, e); + return false; + } + } + + @Override + public boolean exists(String fileId) { + try { + validateFileId(fileId); + } catch (IllegalArgumentException e) { + return false; + } + HeadObjectRequest request = + HeadObjectRequest.builder().bucket(bucket).key(resolveKey(fileId)).build(); + try { + s3Client.headObject(request); + return true; + } catch (NoSuchKeyException e) { + return false; + } catch (S3Exception e) { + if (e.statusCode() == 404) { + return false; + } + log.warn("Error checking existence for file ID: {}", fileId, e); + return false; + } catch (SdkException e) { + log.warn("Error checking existence for file ID: {}", fileId, e); + return false; + } + } + + @Override + public void close() { + if (!ownsClient) { + return; + } + try { + s3Client.close(); + } catch (Exception e) { + log.warn("Error closing S3 client", e); + } + } + + String resolveKey(String fileId) { + return keyPrefix + fileId; + } + + private static void validateFileId(String fileId) { + if (fileId == null || fileId.isBlank()) { + throw new IllegalArgumentException("File ID must not be blank"); + } + if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) { + throw new IllegalArgumentException("Invalid file ID"); + } + } + + private static String normalizePrefix(String prefix) { + if (prefix == null || prefix.isBlank()) { + return ""; + } + String trimmed = prefix.trim(); + if (trimmed.startsWith("/")) { + trimmed = trimmed.substring(1); + } + if (!trimmed.isEmpty() && !trimmed.endsWith("/")) { + trimmed = trimmed + "/"; + } + return trimmed; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3FileStoreConfiguration.java b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3FileStoreConfiguration.java new file mode 100644 index 000000000..b2a516a4e --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/cluster/s3/S3FileStoreConfiguration.java @@ -0,0 +1,37 @@ +package stirling.software.proprietary.cluster.s3; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.cluster.FileStore; +import stirling.software.common.model.ApplicationProperties; + +/** Activates the S3-backed transient {@link FileStore} when {@code cluster.artifactStore=s3}. */ +@Slf4j +@Configuration +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "cluster", name = "artifactStore", havingValue = "s3") +public class S3FileStoreConfiguration { + + private final ApplicationProperties applicationProperties; + + @Bean(destroyMethod = "close") + @ConditionalOnMissingBean + public FileStore fileStore(@Value("${cluster.s3.keyPrefix:transient/}") String keyPrefix) { + ApplicationProperties.Storage.S3 cfg = applicationProperties.getStorage().getS3(); + S3Clients.Bundle bundle = S3Clients.build(cfg, "cluster file store"); + // FileStore has no signed-URL contract; close the unused presigner immediately. + try { + bundle.presigner().close(); + } catch (Exception ignored) { + } + log.info("Cluster FileStore: s3 (bucket={}, keyPrefix={})", cfg.getBucket(), keyPrefix); + return new S3FileStore(bundle.client(), cfg.getBucket(), keyPrefix, true); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/ee/EEAppConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/ee/EEAppConfig.java index afaef2fbd..4fb6a0de3 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/ee/EEAppConfig.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/ee/EEAppConfig.java @@ -49,7 +49,11 @@ public class EEAppConfig { @Profile("security & !saas") @Bean(name = "SSOAutoLogin") public boolean ssoAutoLogin() { - return applicationProperties.getPremium().getProFeatures().isSsoAutoLogin(); + boolean enabled = applicationProperties.getPremium().getProFeatures().isSsoAutoLogin(); + if (enabled) { + licenseKeyChecker.requireProOrEnterprise("premium.proFeatures.ssoAutoLogin=true"); + } + return enabled; } // TODO: Remove post migration diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/ee/LicenseKeyChecker.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/ee/LicenseKeyChecker.java index 0183e58f2..af9ac192d 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/ee/LicenseKeyChecker.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/configuration/ee/LicenseKeyChecker.java @@ -32,7 +32,10 @@ public class LicenseKeyChecker { private final UserLicenseSettingsService licenseSettingsService; - private License premiumEnabledResult = License.NORMAL; + // volatile: written by evaluateLicense() on the @Scheduled refresh thread, read by request + // threads via getPremiumLicenseEnabledResult() / requireProOrEnterprise(). Ensures readers see + // the latest tier rather than a stale cached value. + private volatile License premiumEnabledResult = License.NORMAL; public LicenseKeyChecker( KeygenLicenseVerifier licenseService, @@ -133,4 +136,16 @@ public class LicenseKeyChecker { public License getPremiumLicenseEnabledResult() { return premiumEnabledResult; } + + /** + * Throws {@link IllegalStateException} if the current license is not Pro or Enterprise. Used by + * boot-time gates to fail fast when an operator enables a premium-only setting without a valid + * license. {@code configuredAs} is the human-readable property path (e.g. {@code + * "storage.provider=s3"}) and appears in the exception message. + */ + public void requireProOrEnterprise(String configuredAs) { + if (premiumEnabledResult != License.SERVER && premiumEnabledResult != License.ENTERPRISE) { + throw new IllegalStateException(configuredAs + " requires a Pro or Enterprise license"); + } + } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/storage/config/ClusterStorageGate.java b/app/proprietary/src/main/java/stirling/software/proprietary/storage/config/ClusterStorageGate.java new file mode 100644 index 000000000..07320a9ce --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/storage/config/ClusterStorageGate.java @@ -0,0 +1,95 @@ +package stirling.software.proprietary.storage.config; + +import java.util.Locale; +import java.util.Optional; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +import jakarta.annotation.PostConstruct; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.security.configuration.ee.LicenseKeyChecker; + +/** + * Fails fast at boot if cluster mode is enabled with node-local storage. Validates both {@code + * storage.provider} (persistent uploads) and {@code cluster.artifactStore} (transient job-result + * files): neither may be {@code local} when {@code cluster.enabled=true}. Additionally enforces + * that any S3-backed configuration ({@code storage.provider=s3} or {@code + * cluster.artifactStore=s3}) is accompanied by a valid Pro / Enterprise license. + */ +@Configuration +@RequiredArgsConstructor +@Slf4j +public class ClusterStorageGate { + + private final ApplicationProperties applicationProperties; + private final LicenseKeyChecker licenseKeyChecker; + + @Value("${cluster.enabled:false}") + private boolean clusterEnabled; + + @Value("${cluster.artifactStore:local}") + private String clusterArtifactStore; + + @PostConstruct + void validate() { + // License enforcement runs regardless of cluster.enabled: even a single-node setup that + // selects a remote backend must hold a Pro or higher license. + ApplicationProperties.Storage storage = applicationProperties.getStorage(); + if (storage != null && storage.isEnabled()) { + String provider = normalize(storage.getProvider()); + if ("s3".equals(provider) || "database".equals(provider)) { + licenseKeyChecker.requireProOrEnterprise("storage.provider=" + provider); + } + } + if ("s3".equals(normalize(clusterArtifactStore))) { + licenseKeyChecker.requireProOrEnterprise("cluster.artifactStore=s3"); + } + + if (!clusterEnabled) { + return; + } + if (storage != null && storage.isEnabled()) { + validate( + "storage.provider", + storage.getProvider(), + "Local filesystem storage cannot be shared across cluster nodes." + + " Configure storage.provider=s3 (with storage.s3.bucket /" + + " endpoint / credentials) or storage.provider=database before" + + " enabling clustering."); + } + validate( + "cluster.artifactStore", + clusterArtifactStore, + "Per-node disk cannot back transient job-result files in a multi-node" + + " deployment; downloads would 404 whenever the load balancer routes" + + " a follow-up request to a different node. Configure" + + " cluster.artifactStore=s3 (reuses storage.s3.* config)" + + " before enabling clustering."); + } + + private static String normalize(String value) { + return Optional.ofNullable(value).orElse("local").trim().toLowerCase(Locale.ROOT); + } + + private static void validate(String propertyName, String configuredValue, String remediation) { + String normalized = + Optional.ofNullable(configuredValue) + .orElse("local") + .trim() + .toLowerCase(Locale.ROOT); + if ("local".equals(normalized)) { + throw new IllegalStateException( + "Cluster mode (cluster.enabled=true) is incompatible with " + + propertyName + + "=local. " + + remediation); + } + log.info( + "Cluster storage gate: clusterEnabled=true, {}={} -> OK", propertyName, normalized); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/storage/config/StorageProviderConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/storage/config/StorageProviderConfig.java index d769997fb..e990fa2ce 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/storage/config/StorageProviderConfig.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/storage/config/StorageProviderConfig.java @@ -15,8 +15,11 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.common.configuration.InstallationPathConfig; import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.cluster.s3.S3Clients; +import stirling.software.proprietary.security.configuration.ee.LicenseKeyChecker; import stirling.software.proprietary.storage.provider.DatabaseStorageProvider; import stirling.software.proprietary.storage.provider.LocalStorageProvider; +import stirling.software.proprietary.storage.provider.S3StorageProvider; import stirling.software.proprietary.storage.provider.StorageProvider; import stirling.software.proprietary.storage.repository.StoredFileBlobRepository; @@ -27,8 +30,9 @@ public class StorageProviderConfig { private final ApplicationProperties applicationProperties; private final StoredFileBlobRepository storedFileBlobRepository; + private final LicenseKeyChecker licenseKeyChecker; - @Bean + @Bean(destroyMethod = "close") public StorageProvider storageProvider() { boolean storageEnabled = applicationProperties.getStorage().isEnabled(); String providerName = @@ -37,8 +41,13 @@ public class StorageProviderConfig { .trim() .toLowerCase(Locale.ROOT); if ("database".equals(providerName)) { + licenseKeyChecker.requireProOrEnterprise("storage.provider=database"); return new DatabaseStorageProvider(storedFileBlobRepository); } + if ("s3".equals(providerName)) { + licenseKeyChecker.requireProOrEnterprise("storage.provider=s3"); + return buildS3Provider(applicationProperties.getStorage().getS3()); + } if (!"local".equals(providerName)) { throw new IllegalStateException("Storage provider not supported: " + providerName); } @@ -71,4 +80,9 @@ public class StorageProviderConfig { } return new LocalStorageProvider(basePath); } + + private S3StorageProvider buildS3Provider(ApplicationProperties.Storage.S3 cfg) { + S3Clients.Bundle bundle = S3Clients.build(cfg, "storage provider"); + return new S3StorageProvider(bundle.client(), bundle.presigner(), cfg.getBucket()); + } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/storage/controller/FileStorageController.java b/app/proprietary/src/main/java/stirling/software/proprietary/storage/controller/FileStorageController.java index 6027721a2..4eb299cd2 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/storage/controller/FileStorageController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/storage/controller/FileStorageController.java @@ -1,7 +1,11 @@ package stirling.software.proprietary.storage.controller; +import java.io.IOException; +import java.net.URI; +import java.time.Duration; import java.util.List; import java.util.Locale; +import java.util.Optional; import org.springframework.http.ContentDisposition; import org.springframework.http.HttpHeaders; @@ -25,6 +29,7 @@ import org.springframework.web.server.ResponseStatusException; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import stirling.software.proprietary.security.model.User; import stirling.software.proprietary.storage.model.FileShare; @@ -35,17 +40,22 @@ import stirling.software.proprietary.storage.model.api.ShareLinkMetadataResponse import stirling.software.proprietary.storage.model.api.ShareLinkResponse; import stirling.software.proprietary.storage.model.api.ShareWithUserRequest; import stirling.software.proprietary.storage.model.api.StoredFileResponse; +import stirling.software.proprietary.storage.provider.StorageProvider; import stirling.software.proprietary.storage.service.FileStorageService; @RestController @RequestMapping("/api/v1/storage") @RequiredArgsConstructor +@Slf4j @Tag( name = "File Storage", description = "Stored file management, sharing, and share link operations") public class FileStorageController { + private static final Duration SIGNED_URL_TTL = Duration.ofMinutes(5); + private final FileStorageService fileStorageService; + private final StorageProvider storageProvider; @PostMapping( value = "/files", @@ -91,7 +101,9 @@ public class FileStorageController { User user = fileStorageService.requireAuthenticatedUser(); StoredFile file = fileStorageService.getAccessibleFile(user, fileId); fileStorageService.requireReadAccess(user, file); - return buildFileResponse(file, inline); + Optional> redirect = + tryRedirectToSignedUrl(file, inline); + return redirect.orElseGet(() -> buildFileResponse(file, inline)); } @DeleteMapping("/files/{fileId}") @@ -189,7 +201,9 @@ public class FileStorageController { fileStorageService.requireReadAccess(share); fileStorageService.recordShareAccess(share, authentication, inline); StoredFile file = share.getFile(); - return buildFileResponse(file, inline); + Optional> redirect = + tryRedirectToSignedUrl(file, inline); + return redirect.orElseGet(() -> buildFileResponse(file, inline)); } @GetMapping("/share-links/{token}/metadata") @@ -272,4 +286,34 @@ public class FileStorageController { && authentication.isAuthenticated() && !"anonymousUser".equals(authentication.getPrincipal()); } + + private Optional> tryRedirectToSignedUrl( + StoredFile file, boolean inline) { + if (file == null || file.getStorageKey() == null || file.getStorageKey().isBlank()) { + return Optional.empty(); + } + try { + Optional signed = + storageProvider.signedDownloadUrl( + file.getStorageKey(), + SIGNED_URL_TTL, + inline, + file.getOriginalFilename()); + if (signed.isEmpty()) { + return Optional.empty(); + } + HttpHeaders headers = new HttpHeaders(); + headers.setLocation(signed.get()); + ResponseEntity response = + ResponseEntity.status(HttpStatus.FOUND).headers(headers).build(); + return Optional.of(response); + } catch (IOException e) { + log.warn( + "Failed to create signed download URL for file {} (key: {}), falling back to streaming", + file.getId(), + file.getStorageKey(), + e); + return Optional.empty(); + } + } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/LocalStorageProvider.java b/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/LocalStorageProvider.java index 5790d4372..75f9eb3fd 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/LocalStorageProvider.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/LocalStorageProvider.java @@ -24,6 +24,9 @@ public class LocalStorageProvider implements StorageProvider { @Override public StoredObject store(User owner, MultipartFile file) throws IOException { + if (owner == null || owner.getId() == null) { + throw new IllegalArgumentException("owner.id is required for local storage key"); + } String originalFilename = sanitizeFilename(file.getOriginalFilename()); String storageKey = owner.getId() @@ -77,6 +80,7 @@ public class LocalStorageProvider implements StorageProvider { if (filename == null || filename.isBlank()) { return "file"; } - return Paths.get(filename).getFileName().toString(); + String stripped = Paths.get(filename).getFileName().toString().replaceAll("\\p{Cntrl}", ""); + return stripped.isBlank() ? "file" : stripped; } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/S3StorageProvider.java b/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/S3StorageProvider.java new file mode 100644 index 000000000..a1582458b --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/S3StorageProvider.java @@ -0,0 +1,190 @@ +package stirling.software.proprietary.storage.provider; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Optional; +import java.util.UUID; + +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.web.multipart.MultipartFile; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.security.model.User; + +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; + +/** {@link StorageProvider} backed by an S3-compatible object store. */ +@Slf4j +public class S3StorageProvider implements StorageProvider, AutoCloseable { + + private final S3Client s3Client; + private final S3Presigner s3Presigner; + private final String bucket; + + public S3StorageProvider(S3Client s3Client, S3Presigner s3Presigner, String bucket) { + if (bucket == null || bucket.isBlank()) { + throw new IllegalArgumentException("S3 bucket must be configured"); + } + this.s3Client = s3Client; + this.s3Presigner = s3Presigner; + this.bucket = bucket; + } + + @Override + public StoredObject store(User owner, MultipartFile file) throws IOException { + if (owner == null || owner.getId() == null) { + throw new IllegalArgumentException("owner.id is required for S3 storage key"); + } + String originalFilename = sanitizeFilename(file.getOriginalFilename()); + // Key is opaque ({ownerId}/{uuid}) so non-ASCII filenames don't break vendors that + // restrict key charset (e.g. Supabase Storage returns 400 Invalid key on unicode). + // The display name is preserved in StoredObject.originalFilename and the DB row. + String storageKey = owner.getId() + "/" + UUID.randomUUID(); + + PutObjectRequest.Builder request = + PutObjectRequest.builder().bucket(bucket).key(storageKey); + if (file.getContentType() != null && !file.getContentType().isBlank()) { + request.contentType(file.getContentType()); + } + try (InputStream inputStream = file.getInputStream()) { + s3Client.putObject( + request.build(), RequestBody.fromInputStream(inputStream, file.getSize())); + } catch (SdkException e) { + throw new IOException("Failed to upload object to S3", e); + } + + return StoredObject.builder() + .storageKey(storageKey) + .originalFilename(originalFilename) + .contentType(file.getContentType()) + .sizeBytes(file.getSize()) + .build(); + } + + @Override + public Resource load(String storageKey) throws IOException { + GetObjectRequest request = + GetObjectRequest.builder().bucket(bucket).key(storageKey).build(); + try { + ResponseInputStream stream = s3Client.getObject(request); + long contentLength = + stream.response().contentLength() != null + ? stream.response().contentLength() + : -1; + return new InputStreamResource(stream) { + @Override + public long contentLength() { + return contentLength; + } + }; + } catch (NoSuchKeyException e) { + throw new IOException("File not found", e); + } catch (SdkException e) { + throw new IOException("Failed to load object from S3", e); + } + } + + @Override + public void delete(String storageKey) throws IOException { + try { + s3Client.deleteObject( + DeleteObjectRequest.builder().bucket(bucket).key(storageKey).build()); + } catch (SdkException e) { + throw new IOException("Failed to delete object from S3", e); + } + } + + @Override + public Optional signedDownloadUrl(String storageKey, Duration ttl) throws IOException { + return signedDownloadUrl(storageKey, ttl, false, null); + } + + @Override + public Optional signedDownloadUrl( + String storageKey, Duration ttl, boolean inline, String originalFilename) + throws IOException { + if (storageKey == null || storageKey.isBlank()) { + return Optional.empty(); + } + Duration effectiveTtl = + ttl == null || ttl.isZero() || ttl.isNegative() ? Duration.ofMinutes(5) : ttl; + try { + GetObjectRequest.Builder getBuilder = + GetObjectRequest.builder().bucket(bucket).key(storageKey); + String disposition = buildContentDisposition(inline, originalFilename); + if (disposition != null) { + getBuilder.responseContentDisposition(disposition); + } + GetObjectPresignRequest presignRequest = + GetObjectPresignRequest.builder() + .signatureDuration(effectiveTtl) + .getObjectRequest(getBuilder.build()) + .build(); + PresignedGetObjectRequest presigned = s3Presigner.presignGetObject(presignRequest); + return Optional.of(presigned.url().toURI()); + } catch (SdkException | URISyntaxException e) { + log.warn("Failed to create presigned S3 GET URL for key {}", storageKey, e); + return Optional.empty(); + } + } + + // Returns null when originalFilename is blank; S3 falls back to its own default in that case. + static String buildContentDisposition(boolean inline, String originalFilename) { + if (originalFilename == null || originalFilename.isBlank()) { + return null; + } + // Strip CR/LF and other control chars before path parsing (Paths.get throws on them on + // Windows, and they defeat header parsers). + String stripped = originalFilename.replaceAll("\\p{Cntrl}", ""); + // Use only the basename to avoid leaking directory structure into the header. + int lastSeparator = Math.max(stripped.lastIndexOf('/'), stripped.lastIndexOf('\\')); + if (lastSeparator >= 0) { + stripped = stripped.substring(lastSeparator + 1); + } + if (stripped.isBlank()) { + return null; + } + // Escape per RFC 6266 quoted-string rules. + String escaped = stripped.replace("\\", "\\\\").replace("\"", "\\\""); + return (inline ? "inline" : "attachment") + "; filename=\"" + escaped + "\""; + } + + @Override + public void close() { + try { + s3Presigner.close(); + } catch (Exception e) { + log.warn("Error closing S3 presigner", e); + } + try { + s3Client.close(); + } catch (Exception e) { + log.warn("Error closing S3 client", e); + } + } + + private String sanitizeFilename(String filename) { + if (filename == null || filename.isBlank()) { + return "file"; + } + String stripped = Paths.get(filename).getFileName().toString().replaceAll("\\p{Cntrl}", ""); + return stripped.isBlank() ? "file" : stripped; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/StorageProvider.java b/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/StorageProvider.java index 5433e68ba..cc659c1b9 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/StorageProvider.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/storage/provider/StorageProvider.java @@ -1,16 +1,45 @@ package stirling.software.proprietary.storage.provider; import java.io.IOException; +import java.net.URI; +import java.time.Duration; +import java.util.Optional; import org.springframework.core.io.Resource; import org.springframework.web.multipart.MultipartFile; import stirling.software.proprietary.security.model.User; -public interface StorageProvider { +public interface StorageProvider extends AutoCloseable { StoredObject store(User owner, MultipartFile file) throws IOException; Resource load(String storageKey) throws IOException; void delete(String storageKey) throws IOException; + + /** + * Releases any backend-specific resources. Default no-op so {@link LocalStorageProvider} and + * {@link DatabaseStorageProvider} (which hold no closeable handles) satisfy Spring's + * {@code @Bean(destroyMethod = "close")} signature requirement without ceremony. {@code + * S3StorageProvider} overrides this to close the underlying SDK client + presigner. + */ + @Override + default void close() {} + + /** + * Returns a presigned download URL valid for {@code ttl}, or {@link Optional#empty()} if the + * provider does not support signed URLs (callers fall back to {@link #load(String)}). + */ + default Optional signedDownloadUrl(String storageKey, Duration ttl) throws IOException { + return signedDownloadUrl(storageKey, ttl, false, null); + } + + /** + * Like {@link #signedDownloadUrl(String, Duration)} with explicit Content-Disposition control. + */ + default Optional signedDownloadUrl( + String storageKey, Duration ttl, boolean inline, String originalFilename) + throws IOException { + return Optional.empty(); + } } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3ClientsTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3ClientsTest.java new file mode 100644 index 000000000..50fb7ef18 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3ClientsTest.java @@ -0,0 +1,147 @@ +package stirling.software.proprietary.cluster.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.net.URI; + +import org.junit.jupiter.api.Test; + +import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; +import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; + +class S3ClientsTest { + + @Test + void validateEndpointHost_publicAwsHost_passes() { + assertThatCode( + () -> + S3Clients.validateEndpointHost( + URI.create("https://s3.us-east-1.amazonaws.com"), false)) + .doesNotThrowAnyException(); + } + + @Test + void validateEndpointHost_metadataServiceIp_rejected() { + assertThatThrownBy( + () -> + S3Clients.validateEndpointHost( + URI.create("http://169.254.169.254/"), false)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("allow-private-endpoints"); + } + + @Test + void validateEndpointHost_loopback_rejected() { + assertThatThrownBy( + () -> + S3Clients.validateEndpointHost( + URI.create("http://127.0.0.1:9000/"), false)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void validateEndpointHost_rfc1918Private_rejected() { + assertThatThrownBy( + () -> + S3Clients.validateEndpointHost( + URI.create("http://10.0.0.5:9000/"), false)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void validateEndpointHost_allowPrivateOptIn_bypassesCheck() { + assertThatCode( + () -> + S3Clients.validateEndpointHost( + URI.create("http://169.254.169.254/"), true)) + .doesNotThrowAnyException(); + assertThatCode( + () -> + S3Clients.validateEndpointHost( + URI.create("http://127.0.0.1:9000/"), true)) + .doesNotThrowAnyException(); + } + + @Test + void validateEndpointHost_missingHost_rejected() { + assertThatThrownBy( + () -> + S3Clients.validateEndpointHost( + URI.create("file:///etc/passwd"), false)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("must include a host"); + } + + @Test + void validateEndpointHost_errorMessageNamesTheFlag() { + assertThat( + catchMessage( + () -> + S3Clients.validateEndpointHost( + URI.create("http://192.168.1.10:9000/"), false))) + .contains("storage.s3.allow-private-endpoints"); + } + + // ----- requestChecksumCalculation parsing ----- + + @Test + void parseRequestChecksum_nullOrBlank_defaultsToWhenSupported() { + assertThat(S3Clients.parseRequestChecksum(null)) + .isEqualTo(RequestChecksumCalculation.WHEN_SUPPORTED); + assertThat(S3Clients.parseRequestChecksum("")) + .isEqualTo(RequestChecksumCalculation.WHEN_SUPPORTED); + assertThat(S3Clients.parseRequestChecksum(" ")) + .isEqualTo(RequestChecksumCalculation.WHEN_SUPPORTED); + } + + @Test + void parseRequestChecksum_caseInsensitive_andTrimmed() { + assertThat(S3Clients.parseRequestChecksum("when_required")) + .isEqualTo(RequestChecksumCalculation.WHEN_REQUIRED); + assertThat(S3Clients.parseRequestChecksum(" WHEN_REQUIRED ")) + .isEqualTo(RequestChecksumCalculation.WHEN_REQUIRED); + assertThat(S3Clients.parseRequestChecksum("When_Supported")) + .isEqualTo(RequestChecksumCalculation.WHEN_SUPPORTED); + } + + @Test + void parseRequestChecksum_unknownValue_fallsBackToDefault() { + assertThat(S3Clients.parseRequestChecksum("yes-please")) + .isEqualTo(RequestChecksumCalculation.WHEN_SUPPORTED); + assertThat(S3Clients.parseRequestChecksum("disabled-completely")) + .isEqualTo(RequestChecksumCalculation.WHEN_SUPPORTED); + } + + // ----- responseChecksumValidation parsing ----- + + @Test + void parseResponseChecksum_nullOrBlank_defaultsToWhenSupported() { + assertThat(S3Clients.parseResponseChecksum(null)) + .isEqualTo(ResponseChecksumValidation.WHEN_SUPPORTED); + assertThat(S3Clients.parseResponseChecksum("")) + .isEqualTo(ResponseChecksumValidation.WHEN_SUPPORTED); + } + + @Test + void parseResponseChecksum_explicitWhenRequired_returnedAsEnum() { + assertThat(S3Clients.parseResponseChecksum("WHEN_REQUIRED")) + .isEqualTo(ResponseChecksumValidation.WHEN_REQUIRED); + } + + @Test + void parseResponseChecksum_unknownValue_fallsBackToDefault() { + assertThat(S3Clients.parseResponseChecksum("nope")) + .isEqualTo(ResponseChecksumValidation.WHEN_SUPPORTED); + } + + private static String catchMessage(Runnable r) { + try { + r.run(); + return ""; + } catch (RuntimeException e) { + return e.getMessage() == null ? "" : e.getMessage(); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3FileStoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3FileStoreTest.java new file mode 100644 index 000000000..b407041bd --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3FileStoreTest.java @@ -0,0 +1,253 @@ +package stirling.software.proprietary.cluster.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MinIOContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import stirling.software.common.cluster.FileStore; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; + +@Testcontainers(disabledWithoutDocker = true) +class S3FileStoreTest { + + private static final String BUCKET = "stirling-test-filestore"; + private static final String ACCESS_KEY = "minioadmin"; + private static final String SECRET_KEY = "minioadmin"; + + @Container + static MinIOContainer minio = + new MinIOContainer("minio/minio:latest") + .withUserName(ACCESS_KEY) + .withPassword(SECRET_KEY); + + private static S3Client s3Client; + private static S3FileStore store; + + @BeforeAll + static void setUp() { + URI endpoint = URI.create(minio.getS3URL()); + AwsBasicCredentials creds = AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY); + S3Configuration s3Config = S3Configuration.builder().pathStyleAccessEnabled(true).build(); + + s3Client = + S3Client.builder() + .endpointOverride(endpoint) + .httpClient(UrlConnectionHttpClient.create()) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create(creds)) + .serviceConfiguration(s3Config) + .build(); + + s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build()); + store = new S3FileStore(s3Client, BUCKET, "transient/", false); + } + + @AfterAll + static void tearDown() { + if (store != null) { + store.close(); + } + if (s3Client != null) { + s3Client.close(); + } + } + + @Test + void blankBucket_constructorRejects() { + assertThatThrownBy(() -> new S3FileStore(s3Client, "")) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new S3FileStore(s3Client, null)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void store_thenRetrieve_roundTripsContent() throws IOException { + byte[] payload = "hello cluster s3".getBytes(StandardCharsets.UTF_8); + FileStore.Stored stored = store.store(new ByteArrayInputStream(payload), "foo.txt"); + + assertThat(stored.fileId()).isNotBlank(); + assertThat(stored.size()).isEqualTo(payload.length); + + assertThat(store.exists(stored.fileId())).isTrue(); + assertThat(store.size(stored.fileId())).isEqualTo(payload.length); + assertThat(store.retrieveBytes(stored.fileId())).isEqualTo(payload); + + try (InputStream in = store.retrieve(stored.fileId())) { + assertThat(in.readAllBytes()).isEqualTo(payload); + } + } + + @Test + void store_keysUseConfiguredPrefix() throws IOException { + byte[] payload = "prefixed".getBytes(StandardCharsets.UTF_8); + FileStore.Stored stored = store.store(new ByteArrayInputStream(payload), "p.txt"); + + String prefixed = store.resolveKey(stored.fileId()); + assertThat(prefixed).startsWith("transient/"); + s3Client.headObject(HeadObjectRequest.builder().bucket(BUCKET).key(prefixed).build()); + + assertThatThrownBy( + () -> + s3Client.headObject( + HeadObjectRequest.builder() + .bucket(BUCKET) + .key(stored.fileId()) + .build())) + .isInstanceOfAny( + NoSuchKeyException.class, + software.amazon.awssdk.services.s3.model.S3Exception.class); + } + + @Test + void emptyPrefix_writesAtBucketRoot() throws IOException { + S3FileStore rootStore = new S3FileStore(s3Client, BUCKET, "", false); + byte[] payload = "no-prefix".getBytes(StandardCharsets.UTF_8); + FileStore.Stored stored = rootStore.store(new ByteArrayInputStream(payload), "r.txt"); + assertThat(rootStore.resolveKey(stored.fileId())).isEqualTo(stored.fileId()); + assertThat(rootStore.retrieveBytes(stored.fileId())).isEqualTo(payload); + assertThat(rootStore.delete(stored.fileId())).isTrue(); + } + + @Test + void delete_removesObject_andReturnsTrue() throws IOException { + FileStore.Stored stored = + store.store(new ByteArrayInputStream(new byte[] {1, 2, 3}), "d.bin"); + assertThat(store.delete(stored.fileId())).isTrue(); + assertThat(store.exists(stored.fileId())).isFalse(); + assertThatThrownBy(() -> store.retrieveBytes(stored.fileId())) + .isInstanceOf(IOException.class); + } + + @Test + void delete_unknownKey_isIdempotentReturnsTrue() { + // S3 DeleteObject is idempotent (returns 204 whether or not the object existed). + // The store reflects S3's behaviour rather than racing a HEAD before each DELETE. + assertThat(store.delete("00000000-0000-0000-0000-000000000000")).isTrue(); + } + + @Test + void retrieve_missingKey_throwsIOException() { + assertThatThrownBy(() -> store.retrieveBytes("does-not-exist")) + .isInstanceOf(IOException.class); + assertThatThrownBy(() -> store.retrieve("does-not-exist")).isInstanceOf(IOException.class); + assertThatThrownBy(() -> store.size("does-not-exist")).isInstanceOf(IOException.class); + } + + @Test + void exists_returnsFalseForBlankOrTraversalIds() { + assertThat(store.exists(null)).isFalse(); + assertThat(store.exists("")).isFalse(); + assertThat(store.exists("..")).isFalse(); + assertThat(store.exists("a/b")).isFalse(); + assertThat(store.exists("a\\b")).isFalse(); + } + + @Test + void delete_traversalId_returnsFalseWithoutCall() { + assertThat(store.delete("../etc/passwd")).isFalse(); + assertThat(store.delete("foo/bar")).isFalse(); + } + + @Test + void store_largePayload_streamsViaTempFileWithoutBufferingInMemory() throws IOException { + long payloadSize = 16L * 1024 * 1024; + Path tempDir = Path.of(System.getProperty("java.io.tmpdir")); + long uploadTempsBefore = countS3UploadTemps(tempDir); + + FileStore.Stored stored; + try (InputStream large = new RepeatingInputStream((byte) 0x42, payloadSize)) { + stored = store.store(large, "big.bin"); + } + + assertThat(stored.size()).isEqualTo(payloadSize); + assertThat(store.size(stored.fileId())).isEqualTo(payloadSize); + assertThat(countS3UploadTemps(tempDir)).isEqualTo(uploadTempsBefore); + store.delete(stored.fileId()); + } + + @Test + void store_uploadFailure_stillDeletesTempFile() { + Path tempDir = Path.of(System.getProperty("java.io.tmpdir")); + long uploadTempsBefore = countS3UploadTemps(tempDir); + + // Non-existent bucket causes putObject to fail after the temp file is written, exercising + // the failure-path cleanup in the finally block. + S3FileStore brokenStore = + new S3FileStore(s3Client, "bucket-that-does-not-exist", "transient/", false); + + assertThatThrownBy( + () -> + brokenStore.store( + new ByteArrayInputStream( + "payload".getBytes(StandardCharsets.UTF_8)), + "x.bin")) + .isInstanceOf(IOException.class); + + assertThat(countS3UploadTemps(tempDir)).isEqualTo(uploadTempsBefore); + } + + private static long countS3UploadTemps(Path tempDir) { + try (Stream entries = Files.list(tempDir)) { + return entries.filter(p -> p.getFileName().toString().startsWith("s3-upload-")).count(); + } catch (IOException e) { + return 0L; + } + } + + /** Generates {@code length} bytes of a single value without buffering them in memory. */ + private static final class RepeatingInputStream extends InputStream { + private final byte value; + private long remaining; + + RepeatingInputStream(byte value, long length) { + this.value = value; + this.remaining = length; + } + + @Override + public int read() { + if (remaining <= 0) { + return -1; + } + remaining--; + return value & 0xFF; + } + + @Override + public int read(byte[] b, int off, int len) { + if (remaining <= 0) { + return -1; + } + int toWrite = (int) Math.min(len, remaining); + for (int i = 0; i < toWrite; i++) { + b[off + i] = value; + } + remaining -= toWrite; + return toWrite; + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3VendorComprehensiveTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3VendorComprehensiveTest.java new file mode 100644 index 000000000..d19c9b171 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3VendorComprehensiveTest.java @@ -0,0 +1,779 @@ +package stirling.software.proprietary.cluster.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.springframework.mock.web.MockMultipartFile; + +import stirling.software.common.cluster.FileStore; +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.storage.provider.S3StorageProvider; +import stirling.software.proprietary.storage.provider.StoredObject; + +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; + +/** + * Comprehensive live-vendor test against a real S3-compatible endpoint specified via {@code + * S3_SMOKE_*} env vars. Skipped automatically when {@code S3_SMOKE_ENDPOINT} is not set, so CI is + * not affected. Covers: + * + *
    + *
  • {@code S3StorageProvider} CRUD: store / load / delete / presigned URL + *
  • {@code S3FileStore} CRUD (cluster artifact path) + *
  • Folder semantics simulated via key prefixes (matches production usage) + *
  • Negative paths: wrong secret, missing bucket, missing key, traversal IDs + *
  • Edge cases: zero-byte, unicode filename, multi-megabyte streaming + *
  • Configuration guards: SSRF endpoint rejection, bucket validation + *
+ * + * Every uploaded key is tracked and removed in {@link #cleanUp} so re-running against the same + * bucket leaves no residue. + */ +@EnabledIfEnvironmentVariable(named = "S3_SMOKE_ENDPOINT", matches = ".+") +class S3VendorComprehensiveTest { + + private static final String PREFIX = "stirling-comprehensive/" + UUID.randomUUID() + "/"; + + private static ApplicationProperties.Storage.S3 cfg; + private static S3Clients.Bundle bundle; + private static S3StorageProvider provider; + private static String bucket; + private static String vendorLabel; + private static User owner; + + private static final List keysToCleanup = + Collections.synchronizedList(new ArrayList<>()); + + @BeforeAll + static void setUp() { + cfg = configFromEnv(); + bucket = cfg.getBucket(); + vendorLabel = System.getenv().getOrDefault("S3_SMOKE_LABEL", "external"); + bundle = S3Clients.build(cfg, "comprehensive[" + vendorLabel + "]"); + provider = new S3StorageProvider(bundle.client(), bundle.presigner(), bucket); + + owner = new User(); + owner.setId(7L); + owner.setUsername("comprehensive-tester"); + } + + @AfterAll + static void cleanUp() { + if (bundle != null) { + for (String key : keysToCleanup) { + try { + bundle.client().deleteObject(d -> d.bucket(bucket).key(key)); + } catch (Exception e) { + // Best-effort cleanup; ignore. + } + } + try { + provider.close(); + } catch (Exception ignored) { + } + bundle.close(); + } + } + + private static String track(String key) { + keysToCleanup.add(key); + return key; + } + + private static ApplicationProperties.Storage.S3 configFromEnv() { + ApplicationProperties.Storage.S3 c = new ApplicationProperties.Storage.S3(); + c.setEndpoint(System.getenv("S3_SMOKE_ENDPOINT")); + c.setBucket(requireEnv("S3_SMOKE_BUCKET")); + c.setRegion(System.getenv().getOrDefault("S3_SMOKE_REGION", "us-east-1")); + c.setAccessKey(requireEnv("S3_SMOKE_KEY")); + c.setSecretKey(requireEnv("S3_SMOKE_SECRET")); + c.setPathStyleAccess( + Boolean.parseBoolean(System.getenv().getOrDefault("S3_SMOKE_PATHSTYLE", "false"))); + c.setAllowPrivateEndpoints(false); + return c; + } + + private static String requireEnv(String name) { + String value = System.getenv(name); + if (value == null || value.isBlank()) { + throw new IllegalStateException(name + " env var must be set"); + } + return value; + } + + // ========================================================================================== + // FILE CRUD via S3StorageProvider (user-uploaded files) + // ========================================================================================== + + @Test + void provider_store_thenLoad_matchesBytes() throws IOException { + byte[] payload = ("provider-roundtrip-" + vendorLabel).getBytes(StandardCharsets.UTF_8); + MockMultipartFile file = new MockMultipartFile("file", "doc.txt", "text/plain", payload); + + StoredObject obj = provider.store(owner, file); + track(obj.getStorageKey()); + + assertThat(obj.getStorageKey()).isNotBlank(); + assertThat(obj.getSizeBytes()).isEqualTo(payload.length); + assertThat(provider.load(obj.getStorageKey()).getInputStream().readAllBytes()) + .isEqualTo(payload); + } + + @Test + void provider_delete_removesObject() throws IOException { + byte[] payload = "delete-me".getBytes(StandardCharsets.UTF_8); + MockMultipartFile file = new MockMultipartFile("file", "x.txt", "text/plain", payload); + StoredObject obj = provider.store(owner, file); + track(obj.getStorageKey()); + + provider.delete(obj.getStorageKey()); + + assertThatThrownBy(() -> provider.load(obj.getStorageKey())) + .isInstanceOf(IOException.class); + } + + @Test + void provider_load_missingKey_throws() { + assertThatThrownBy(() -> provider.load(PREFIX + "does-not-exist")) + .isInstanceOf(IOException.class); + } + + @Test + void provider_presignedDownload_returnsBytesOverHttp() throws Exception { + byte[] payload = "presign me".getBytes(StandardCharsets.UTF_8); + MockMultipartFile file = new MockMultipartFile("file", "p.txt", "text/plain", payload); + StoredObject obj = provider.store(owner, file); + track(obj.getStorageKey()); + + java.util.Optional url = + provider.signedDownloadUrl(obj.getStorageKey(), Duration.ofMinutes(5)); + assertThat(url).isPresent(); + + HttpResponse resp = + HttpClient.newHttpClient() + .send( + HttpRequest.newBuilder(url.get()).GET().build(), + HttpResponse.BodyHandlers.ofByteArray()); + + assertThat(resp.statusCode()).isEqualTo(200); + assertThat(resp.body()).isEqualTo(payload); + } + + @Test + void provider_store_zeroBytes_isAccepted() throws IOException { + MockMultipartFile empty = + new MockMultipartFile("file", "empty.txt", "text/plain", new byte[0]); + StoredObject obj = provider.store(owner, empty); + track(obj.getStorageKey()); + + assertThat(obj.getSizeBytes()).isZero(); + assertThat(provider.load(obj.getStorageKey()).getInputStream().readAllBytes()) + .isEqualTo(new byte[0]); + } + + @Test + void provider_store_unicodeFilename_yieldsOpaqueAsciiKey_andPreservesNameForDisplay() + throws IOException { + // Regression: pre-fix, the storage key embedded the filename verbatim, which Supabase + // rejected with 400 Invalid key. Post-fix, the key is {ownerId}/{uuid} (ASCII-only) + // and the original unicode name lives on StoredObject.originalFilename. + String unicodeName = "résumé-日本語-é.pdf"; + byte[] payload = "u".getBytes(StandardCharsets.UTF_8); + MockMultipartFile file = + new MockMultipartFile("file", unicodeName, "application/pdf", payload); + + StoredObject obj = provider.store(owner, file); + track(obj.getStorageKey()); + + assertThat(obj.getStorageKey()).matches("[0-9]+/[0-9a-fA-F-]+"); + assertThat(obj.getStorageKey()) + .isEqualTo( + new String( + obj.getStorageKey().getBytes(StandardCharsets.US_ASCII), + StandardCharsets.US_ASCII)); + assertThat(obj.getOriginalFilename()).isEqualTo(unicodeName); + assertThat(provider.load(obj.getStorageKey()).getInputStream().readAllBytes()) + .isEqualTo(payload); + } + + // ========================================================================================== + // Concurrency, overwrite, TTL expiry (added after initial run surfaced the unicode bug) + // ========================================================================================== + + @Test + void provider_concurrent10Uploads_allSucceedWithDistinctKeys() throws Exception { + int n = 10; + java.util.concurrent.ExecutorService pool = + java.util.concurrent.Executors.newFixedThreadPool(n); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < n; i++) { + final int idx = i; + futures.add( + pool.submit( + () -> { + byte[] payload = + ("concurrent-" + idx).getBytes(StandardCharsets.UTF_8); + MockMultipartFile f = + new MockMultipartFile( + "file", + "c-" + idx + ".txt", + "text/plain", + payload); + StoredObject obj = provider.store(owner, f); + track(obj.getStorageKey()); + return obj; + })); + } + + java.util.Set keys = new java.util.HashSet<>(); + for (java.util.concurrent.Future fut : futures) { + StoredObject obj = fut.get(30, java.util.concurrent.TimeUnit.SECONDS); + assertThat(keys.add(obj.getStorageKey())) + .as("distinct key for each parallel upload") + .isTrue(); + assertThat(provider.load(obj.getStorageKey()).getInputStream().readAllBytes()) + .isNotEmpty(); + } + } finally { + pool.shutdownNow(); + } + } + + @Test + void sameKey_overwrite_returnsLatestPayload() { + String key = PREFIX + "overwrite-" + UUID.randomUUID() + ".txt"; + track(key); + + byte[] first = "FIRST".getBytes(StandardCharsets.UTF_8); + byte[] second = "SECOND".getBytes(StandardCharsets.UTF_8); + + bundle.client().putObject(p -> p.bucket(bucket).key(key), RequestBody.fromBytes(first)); + bundle.client().putObject(p -> p.bucket(bucket).key(key), RequestBody.fromBytes(second)); + + assertThat(getRaw(key)).isEqualTo(second); + } + + @Test + void presignedDownload_afterTtlExpiry_returns403() throws Exception { + String key = PREFIX + "presign-expiry-" + UUID.randomUUID() + ".txt"; + byte[] payload = "presign expiry".getBytes(StandardCharsets.UTF_8); + track(putRaw(key, "presign expiry")); + + // 2-second TTL, then wait long enough that any vendor clock skew tolerance is also past. + PresignedGetObjectRequest presigned = + bundle.presigner() + .presignGetObject( + GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofSeconds(2)) + .getObjectRequest(g -> g.bucket(bucket).key(key)) + .build()); + + // Confirm it works while valid - rules out unrelated failures. + HttpResponse ok = + HttpClient.newHttpClient() + .send( + HttpRequest.newBuilder(presigned.url().toURI()).GET().build(), + HttpResponse.BodyHandlers.ofByteArray()); + assertThat(ok.statusCode()).isEqualTo(200); + assertThat(ok.body()).isEqualTo(payload); + + Thread.sleep(5_000); + + HttpResponse expired = + HttpClient.newHttpClient() + .send( + HttpRequest.newBuilder(presigned.url().toURI()).GET().build(), + HttpResponse.BodyHandlers.ofByteArray()); + assertThat(expired.statusCode()) + .as("presigned URL must be rejected after TTL expires") + .isIn(400, 403); + } + + @Test + void provider_store_4MBPayload_streams() throws IOException { + byte[] payload = new byte[4 * 1024 * 1024]; + java.util.Arrays.fill(payload, (byte) 0x42); + MockMultipartFile file = + new MockMultipartFile("file", "big.bin", "application/octet-stream", payload); + + StoredObject obj = provider.store(owner, file); + track(obj.getStorageKey()); + + assertThat(obj.getSizeBytes()).isEqualTo(payload.length); + assertThat(provider.load(obj.getStorageKey()).getInputStream().readAllBytes()) + .isEqualTo(payload); + } + + // ========================================================================================== + // FILE CRUD via S3FileStore (cluster artifact path) + // ========================================================================================== + + @Test + void fileStore_storeAndRetrieve_roundTrip() throws IOException { + S3FileStore store = new S3FileStore(bundle.client(), bucket, PREFIX + "fs/", false); + byte[] payload = "filestore round trip".getBytes(StandardCharsets.UTF_8); + + FileStore.Stored stored = store.store(new ByteArrayInputStream(payload), "rt.txt"); + track(store.resolveKey(stored.fileId())); + + assertThat(store.size(stored.fileId())).isEqualTo(payload.length); + assertThat(store.retrieveBytes(stored.fileId())).isEqualTo(payload); + assertThat(store.exists(stored.fileId())).isTrue(); + } + + @Test + void fileStore_delete_returnsTrue_andExistsFalseAfter() throws IOException { + S3FileStore store = new S3FileStore(bundle.client(), bucket, PREFIX + "fs/", false); + FileStore.Stored stored = store.store(new ByteArrayInputStream("x".getBytes()), "del.txt"); + + assertThat(store.delete(stored.fileId())).isTrue(); + assertThat(store.exists(stored.fileId())).isFalse(); + } + + @Test + void fileStore_retrieveBytes_missingKey_throws() { + S3FileStore store = new S3FileStore(bundle.client(), bucket, PREFIX + "fs/", false); + assertThatThrownBy(() -> store.retrieveBytes("does-not-exist")) + .isInstanceOf(IOException.class); + } + + @Test + void fileStore_rejectsTraversalId() { + S3FileStore store = new S3FileStore(bundle.client(), bucket, PREFIX + "fs/", false); + assertThat(store.exists("..")).isFalse(); + assertThat(store.delete("../etc/passwd")).isFalse(); + assertThat(store.exists("a/b")).isFalse(); + assertThat(store.exists("a\\b")).isFalse(); + } + + // ========================================================================================== + // Folder semantics simulated via key prefixes + // ========================================================================================== + + @Test + void folderPrefix_isolatesObjects_andDeleteByPrefixDoesNotTouchRoot() throws IOException { + // Two "folders" + a root object - all reuse the test PREFIX so cleanup catches them. + String folderA = PREFIX + "folder-A/"; + String folderB = PREFIX + "folder-B/"; + String rootObj = PREFIX + "root-" + UUID.randomUUID() + ".txt"; + + track(putRaw(folderA + "file-1.txt", "in-A")); + track(putRaw(folderA + "file-2.txt", "in-A2")); + track(putRaw(folderB + "file-1.txt", "in-B")); + track(putRaw(rootObj, "at-root")); + + // "Delete folder A": delete every key under folderA prefix + deleteAllUnderPrefix(folderA); + + // Verify A is empty, B and root untouched + assertThat(headOrNull(folderA + "file-1.txt")).isNull(); + assertThat(headOrNull(folderB + "file-1.txt")).isNotNull(); + assertThat(headOrNull(rootObj)).isNotNull(); + } + + @Test + void moveBetweenFolders_viaCopyAndDelete_preservesContent() throws Exception { + String oldKey = PREFIX + "move-old/" + UUID.randomUUID() + ".txt"; + String newKey = PREFIX + "move-new/" + UUID.randomUUID() + ".txt"; + byte[] payload = "moveable".getBytes(StandardCharsets.UTF_8); + + track(oldKey); + track(newKey); + bundle.client() + .putObject(p -> p.bucket(bucket).key(oldKey), RequestBody.fromBytes(payload)); + + // Simulate move: server-side copy + delete original. + bundle.client() + .copyObject( + c -> + c.sourceBucket(bucket) + .sourceKey(oldKey) + .destinationBucket(bucket) + .destinationKey(newKey)); + bundle.client().deleteObject(d -> d.bucket(bucket).key(oldKey)); + + assertThat(headOrNull(oldKey)).isNull(); + assertThat(getRaw(newKey)).isEqualTo(payload); + } + + // ========================================================================================== + // Negative: wrong settings / wrong creds + // ========================================================================================== + + @Test + void wrongSecret_throwsOnFirstOperation() { + ApplicationProperties.Storage.S3 bad = configFromEnv(); + bad.setSecretKey("definitely-not-the-real-secret-" + UUID.randomUUID()); + + try (S3Clients.Bundle badBundle = S3Clients.build(bad, "wrong-secret")) { + assertThatThrownBy(() -> badBundle.client().headBucket(h -> h.bucket(bucket))) + .isInstanceOf(S3Exception.class) + .satisfies(e -> assertThat(((S3Exception) e).statusCode()).isIn(401, 403, 400)); + } + } + + @Test + void nonExistentBucket_throwsOnHeadOrPut() { + String fakeBucket = "stirling-no-such-bucket-" + UUID.randomUUID(); + assertThatThrownBy(() -> bundle.client().headBucket(h -> h.bucket(fakeBucket))) + .isInstanceOfAny(NoSuchBucketException.class, S3Exception.class); + } + + @Test + void blankBucket_atBuildTime_throwsIllegalState() { + ApplicationProperties.Storage.S3 bad = configFromEnv(); + bad.setBucket(""); + assertThatThrownBy(() -> S3Clients.build(bad, "blank-bucket")) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("bucket"); + } + + @Test + void invalidEndpointUri_atBuildTime_throwsIllegalState() { + ApplicationProperties.Storage.S3 bad = configFromEnv(); + bad.setEndpoint("not a valid uri ::::"); + assertThatThrownBy(() -> S3Clients.build(bad, "bad-uri")) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void privateEndpoint_withoutOptIn_atBuildTime_throwsIllegalState() { + ApplicationProperties.Storage.S3 bad = configFromEnv(); + bad.setEndpoint("http://127.0.0.1:9000"); + bad.setAllowPrivateEndpoints(false); + assertThatThrownBy(() -> S3Clients.build(bad, "loopback")) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("private"); + } + + @Test + void getMissingKey_returnsNoSuchKey() { + String missing = PREFIX + "missing-" + UUID.randomUUID(); + assertThatThrownBy(() -> bundle.client().getObject(g -> g.bucket(bucket).key(missing))) + .isInstanceOfAny(NoSuchKeyException.class, S3Exception.class); + } + + // ========================================================================================== + // Bundle lifecycle + // ========================================================================================== + + @Test + void bundleClose_isIdempotent() { + ApplicationProperties.Storage.S3 c = configFromEnv(); + S3Clients.Bundle b = S3Clients.build(c, "lifecycle"); + b.close(); + b.close(); // should not throw + } + + // ========================================================================================== + // Internal helpers (using the bundle directly for prefix/folder simulation) + // ========================================================================================== + + private String putRaw(String key, String body) { + bundle.client() + .putObject( + p -> p.bucket(bucket).key(key), + RequestBody.fromBytes(body.getBytes(StandardCharsets.UTF_8))); + return key; + } + + private byte[] getRaw(String key) { + return bundle.client().getObjectAsBytes(g -> g.bucket(bucket).key(key)).asByteArray(); + } + + private Object headOrNull(String key) { + try { + return bundle.client().headObject(h -> h.bucket(bucket).key(key)); + } catch (Exception e) { + return null; + } + } + + private byte[] tryGetBytes(String key) { + try { + return bundle.client().getObjectAsBytes(g -> g.bucket(bucket).key(key)).asByteArray(); + } catch (Exception e) { + return null; + } + } + + private void deleteAllUnderPrefix(String prefix) { + var listing = bundle.client().listObjectsV2(l -> l.bucket(bucket).prefix(prefix)); + for (var obj : listing.contents()) { + bundle.client().deleteObject(d -> d.bucket(bucket).key(obj.key())); + } + } + + // ========================================================================================== + // Key edge cases: leading/trailing/double slash, length, URL-special chars + // ========================================================================================== + + @Test + void key_trailingSlash_storesAsZeroByteFolderMarker() { + String key = PREFIX + "folder-marker-" + UUID.randomUUID() + "/"; + track(key); + + // S3 spec: trailing slash is legal and creates a 0-byte "folder marker" object. + // Some vendors normalize it away; capture either behavior. + bundle.client() + .putObject(p -> p.bucket(bucket).key(key), RequestBody.fromBytes(new byte[0])); + Object head = headOrNull(key); + // Either: vendor accepts the marker (head is non-null) or normalizes to bare key. + assertThat(head != null || headOrNull(key.substring(0, key.length() - 1)) != null) + .as("vendor should either accept trailing-slash marker or normalize to bare key") + .isTrue(); + } + + @Test + void key_doubleSlash_normalizedOrStoredVerbatim() { + String key = PREFIX + "double//slash-" + UUID.randomUUID() + ".txt"; + track(key); + bundle.client() + .putObject( + p -> p.bucket(bucket).key(key), + RequestBody.fromBytes("ds".getBytes(StandardCharsets.UTF_8))); + + // Either GET-with-the-exact-key works, or vendor normalized -> single-slash form works. + String alt = key.replace("//", "/"); + track(alt); + byte[] viaExact = tryGetBytes(key); + byte[] viaNormalized = tryGetBytes(alt); + assertThat(viaExact != null || viaNormalized != null) + .as("either exact double-slash key or normalized single-slash form must return") + .isTrue(); + } + + @Test + void key_200Chars_isStoredAndRetrievable() { + // Stirling production keys are ~45 chars ({ownerId}/{uuid}). 200 chars exceeds that by + // ~5x but stays inside every vendor's documented limit. The S3 spec max is 1024 bytes + // but some vendors (Supabase) impose stricter caps (~250-byte total path including + // bucket prefix - 1000 chars fails with KeyTooLongError). + StringBuilder sb = new StringBuilder(PREFIX + "long/"); + while (sb.length() < 200) { + sb.append("abcdefghij"); + } + String longKey = sb.substring(0, 200); + track(longKey); + + byte[] payload = "long-key".getBytes(StandardCharsets.UTF_8); + bundle.client() + .putObject(p -> p.bucket(bucket).key(longKey), RequestBody.fromBytes(payload)); + assertThat(getRaw(longKey)).isEqualTo(payload); + } + + @Test + void key_safeSpecialChars_areSignedAndRetrievableViaSdk() { + // Restrict to chars every S3-compatible vendor accepts: dot, dash, underscore. + // Stirling's production key format ({ownerId}/{uuid}) is even narrower; this test + // confirms the SDK SigV4 signer copes with slightly more exotic ASCII-safe keys. + // Note: Supabase rejects keys containing space / + / ? / & / # ("400 Invalid key"), + // see documentsVendorKeyRestrictions_tolerantTest for that documentation. + String key = + PREFIX + + "safe-special/" + + UUID.randomUUID() + + "_segment.with-dots.and_underscores.txt"; + track(key); + + bundle.client() + .putObject( + p -> p.bucket(bucket).key(key), + RequestBody.fromBytes("safe".getBytes(StandardCharsets.UTF_8))); + assertThat(getRaw(key)).isEqualTo("safe".getBytes(StandardCharsets.UTF_8)); + } + + @Test + void documentsVendorKeyRestrictions_tolerantTest() { + // Documents - rather than enforces - which key characters cause vendor rejection. + // Stirling production code is safe because S3StorageProvider always emits an + // ASCII-safe UUID-only key. If you ever change that, this test becomes a canary. + // AWS S3 and MinIO accept all of these; Supabase rejects all of them with 400. + String[] suspiciousKeys = { + PREFIX + "with space.txt", + PREFIX + "with+plus.txt", + PREFIX + "with#hash.txt", + PREFIX + "with?question.txt", + PREFIX + "with&.txt", + }; + int accepted = 0; + int rejected = 0; + for (String k : suspiciousKeys) { + track(k); + try { + bundle.client() + .putObject( + p -> p.bucket(bucket).key(k), + RequestBody.fromBytes("x".getBytes(StandardCharsets.UTF_8))); + accepted++; + } catch (S3Exception e) { + assertThat(e.statusCode()) + .as("vendor rejection must be a clean 4xx, not a signature mismatch") + .isBetween(400, 499); + rejected++; + } + } + assertThat(accepted + rejected).isEqualTo(suspiciousKeys.length); + } + + // ========================================================================================== + // Presigned-URL: TTL bounds + Content-Disposition behavior (Stirling uses this for shares) + // ========================================================================================== + + @Test + void presignedGet_ttlExceeding7Days_isRejectedAtSigningTime() { + String key = PREFIX + "ttl-overflow-" + UUID.randomUUID() + ".txt"; + track(putRaw(key, "x")); + + // SigV4 caps presigned URL TTL at 7 days. SDK should refuse to sign anything larger. + assertThatThrownBy( + () -> + bundle.presigner() + .presignGetObject( + GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofDays(8)) + .getObjectRequest( + g -> g.bucket(bucket).key(key)) + .build())) + .isInstanceOfAny(IllegalArgumentException.class, RuntimeException.class); + } + + @Test + void provider_signedDownloadUrl_attachmentDisposition_endsWithAttachmentHeader() + throws Exception { + byte[] payload = "attach me".getBytes(StandardCharsets.UTF_8); + MockMultipartFile file = + new MockMultipartFile("file", "report.pdf", "application/pdf", payload); + StoredObject obj = provider.store(owner, file); + track(obj.getStorageKey()); + + java.util.Optional url = + provider.signedDownloadUrl( + obj.getStorageKey(), Duration.ofMinutes(2), false, "report.pdf"); + assertThat(url).isPresent(); + + HttpResponse resp = + HttpClient.newHttpClient() + .send( + HttpRequest.newBuilder(url.get()).GET().build(), + HttpResponse.BodyHandlers.ofByteArray()); + + assertThat(resp.statusCode()).isEqualTo(200); + // Supabase + AWS both honor response-content-disposition query param. + assertThat(resp.headers().firstValue("content-disposition").orElse("")) + .as("vendor must honor response-content-disposition override in presigned URL") + .startsWith("attachment"); + } + + @Test + void provider_signedDownloadUrl_inlineDisposition_endsWithInlineHeader() throws Exception { + byte[] payload = "inline".getBytes(StandardCharsets.UTF_8); + MockMultipartFile file = + new MockMultipartFile("file", "preview.pdf", "application/pdf", payload); + StoredObject obj = provider.store(owner, file); + track(obj.getStorageKey()); + + java.util.Optional url = + provider.signedDownloadUrl( + obj.getStorageKey(), Duration.ofMinutes(2), true, "preview.pdf"); + assertThat(url).isPresent(); + + HttpResponse resp = + HttpClient.newHttpClient() + .send( + HttpRequest.newBuilder(url.get()).GET().build(), + HttpResponse.BodyHandlers.ofByteArray()); + + assertThat(resp.statusCode()).isEqualTo(200); + assertThat(resp.headers().firstValue("content-disposition").orElse("")) + .as("inline=true must set 'inline' disposition") + .startsWith("inline"); + } + + // ========================================================================================== + // List pagination + HEAD missing semantics + // ========================================================================================== + + @Test + void listObjectsV2_paginationWithMaxKeys_returnsContinuationToken() { + // Stage 3 objects under a unique sub-prefix. + String prefix = PREFIX + "page-" + UUID.randomUUID() + "/"; + for (int i = 0; i < 3; i++) { + track(putRaw(prefix + "obj-" + i, "p" + i)); + } + + var first = bundle.client().listObjectsV2(l -> l.bucket(bucket).prefix(prefix).maxKeys(1)); + assertThat(first.contents()).hasSize(1); + assertThat(first.isTruncated()).isTrue(); + assertThat(first.nextContinuationToken()).isNotBlank(); + + var second = + bundle.client() + .listObjectsV2( + l -> + l.bucket(bucket) + .prefix(prefix) + .maxKeys(2) + .continuationToken(first.nextContinuationToken())); + assertThat(second.contents()).hasSize(2); + assertThat(second.isTruncated()).isFalse(); + } + + @Test + void headObject_missingKey_throwsNoSuchKeyOr404() { + String missing = PREFIX + "head-missing-" + UUID.randomUUID(); + assertThatThrownBy(() -> bundle.client().headObject(h -> h.bucket(bucket).key(missing))) + .isInstanceOf(S3Exception.class) + .satisfies(e -> assertThat(((S3Exception) e).statusCode()).isEqualTo(404)); + } + + /** + * Presigned-URL test scaffolding for parity with the smoke test (covers the SDK presign path). + */ + @Test + void presignGetObject_independentOfProvider_returnsBytes() throws Exception { + String key = PREFIX + "presign-direct-" + UUID.randomUUID() + ".txt"; + byte[] payload = "direct presign".getBytes(StandardCharsets.UTF_8); + track(putRaw(key, "direct presign")); + + PresignedGetObjectRequest presigned = + bundle.presigner() + .presignGetObject( + GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(2)) + .getObjectRequest(g -> g.bucket(bucket).key(key)) + .build()); + + HttpResponse resp = + HttpClient.newHttpClient() + .send( + HttpRequest.newBuilder(presigned.url().toURI()).GET().build(), + HttpResponse.BodyHandlers.ofByteArray()); + + assertThat(resp.statusCode()).isEqualTo(200); + assertThat(resp.body()).isEqualTo(payload); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3VendorSmokeTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3VendorSmokeTest.java new file mode 100644 index 000000000..f7178578e --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/cluster/s3/S3VendorSmokeTest.java @@ -0,0 +1,161 @@ +package stirling.software.proprietary.cluster.s3; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.ByteArrayInputStream; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import stirling.software.common.cluster.FileStore; +import stirling.software.common.model.ApplicationProperties; + +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; + +/** + * End-to-end smoke against the full {@link S3Clients#build} path. Defaults to a LocalStack + * container so it runs in CI; if {@code S3_SMOKE_ENDPOINT} is set, swaps in a real vendor (AWS / + * Supabase / R2 / MinIO over network) to validate live signing + DNS. + */ +@Testcontainers(disabledWithoutDocker = true) +class S3VendorSmokeTest { + + private static LocalStackContainer localstack; + private static S3Clients.Bundle bundle; + private static String bucket; + private static String vendorLabel; + + @BeforeAll + static void setUp() { + ApplicationProperties.Storage.S3 cfg = new ApplicationProperties.Storage.S3(); + String envEndpoint = System.getenv("S3_SMOKE_ENDPOINT"); + + if (envEndpoint != null && !envEndpoint.isBlank()) { + vendorLabel = System.getenv().getOrDefault("S3_SMOKE_LABEL", "external"); + cfg.setEndpoint(envEndpoint); + cfg.setBucket(requireEnv("S3_SMOKE_BUCKET")); + cfg.setRegion(System.getenv().getOrDefault("S3_SMOKE_REGION", "us-east-1")); + cfg.setAccessKey(requireEnv("S3_SMOKE_KEY")); + cfg.setSecretKey(requireEnv("S3_SMOKE_SECRET")); + cfg.setPathStyleAccess( + Boolean.parseBoolean( + System.getenv().getOrDefault("S3_SMOKE_PATHSTYLE", "false"))); + cfg.setAllowPrivateEndpoints( + Boolean.parseBoolean( + System.getenv().getOrDefault("S3_SMOKE_ALLOWPRIVATE", "false"))); + } else { + vendorLabel = "localstack"; + localstack = + new LocalStackContainer(DockerImageName.parse("localstack/localstack:3.8")) + .withServices(LocalStackContainer.Service.S3); + localstack.start(); + cfg.setEndpoint( + localstack.getEndpointOverride(LocalStackContainer.Service.S3).toString()); + cfg.setBucket("stirling-smoke"); + cfg.setRegion(localstack.getRegion()); + cfg.setAccessKey(localstack.getAccessKey()); + cfg.setSecretKey(localstack.getSecretKey()); + // Exercise virtual-hosted addressing where possible. LocalStack supports both; + // path-style remains covered by the MinIO suite. + cfg.setPathStyleAccess(false); + // Required: localhost is a loopback address and would otherwise be rejected. + cfg.setAllowPrivateEndpoints(true); + } + + bundle = S3Clients.build(cfg, "vendor-smoke[" + vendorLabel + "]"); + bucket = cfg.getBucket(); + ensureBucketExists(bucket); + } + + @AfterAll + static void tearDown() { + if (bundle != null) { + bundle.close(); + } + if (localstack != null) { + localstack.stop(); + } + } + + @Test + void s3FileStore_roundTripsContentAgainstVendor() throws Exception { + S3FileStore store = new S3FileStore(bundle.client(), bucket, "smoke/", false); + byte[] payload = ("hello from " + vendorLabel).getBytes(StandardCharsets.UTF_8); + + FileStore.Stored stored = + store.store(new ByteArrayInputStream(payload), "smoke-payload.txt"); + try { + assertThat(stored.size()).isEqualTo(payload.length); + assertThat(store.exists(stored.fileId())).isTrue(); + assertThat(store.size(stored.fileId())).isEqualTo(payload.length); + assertThat(store.retrieveBytes(stored.fileId())).isEqualTo(payload); + } finally { + assertThat(store.delete(stored.fileId())).isTrue(); + assertThat(store.exists(stored.fileId())).isFalse(); + } + } + + @Test + void presignedGet_downloadsContentOverHttp() throws Exception { + String key = "smoke/presign-" + System.currentTimeMillis() + ".txt"; + byte[] payload = ("presigned by " + vendorLabel).getBytes(StandardCharsets.UTF_8); + + bundle.client().putObject(p -> p.bucket(bucket).key(key), RequestBody.fromBytes(payload)); + try { + PresignedGetObjectRequest presigned = + bundle.presigner() + .presignGetObject( + GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(5)) + .getObjectRequest(g -> g.bucket(bucket).key(key)) + .build()); + + HttpResponse resp = + HttpClient.newHttpClient() + .send( + HttpRequest.newBuilder(presigned.url().toURI()).GET().build(), + HttpResponse.BodyHandlers.ofByteArray()); + + assertThat(resp.statusCode()).isEqualTo(200); + assertThat(resp.body()).isEqualTo(payload); + } finally { + bundle.client().deleteObject(d -> d.bucket(bucket).key(key)); + } + } + + private static String requireEnv(String name) { + String value = System.getenv(name); + if (value == null || value.isBlank()) { + throw new IllegalStateException( + name + " env var must be set when S3_SMOKE_ENDPOINT is set"); + } + return value; + } + + private static void ensureBucketExists(String b) { + try { + bundle.client().headBucket(h -> h.bucket(b)); + } catch (S3Exception e) { + if (e.statusCode() == 404 || e.statusCode() == 301 || e.statusCode() == 400) { + try { + bundle.client().createBucket(c -> c.bucket(b)); + } catch (S3Exception ignored) { + // Bucket already exists or vendor disallows runtime create (Supabase/R2 often + // require pre-create). Caller is expected to have pre-created it in that case. + } + } + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/configuration/ee/EEAppConfigTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/configuration/ee/EEAppConfigTest.java new file mode 100644 index 000000000..9e5f4165b --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/configuration/ee/EEAppConfigTest.java @@ -0,0 +1,59 @@ +package stirling.software.proprietary.security.configuration.ee; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; + +import stirling.software.common.model.ApplicationProperties; + +class EEAppConfigTest { + + @Test + void ssoAutoLogin_disabled_returnsFalse_andDoesNotConsultLicense() { + ApplicationProperties props = new ApplicationProperties(); + props.getPremium().getProFeatures().setSsoAutoLogin(false); + LicenseKeyChecker checker = mock(LicenseKeyChecker.class); + + EEAppConfig cfg = new EEAppConfig(props, checker); + + assertThat(cfg.ssoAutoLogin()).isFalse(); + verifyNoInteractions(checker); + } + + @Test + void ssoAutoLogin_enabled_withProLicense_returnsTrue() { + ApplicationProperties props = new ApplicationProperties(); + props.getPremium().getProFeatures().setSsoAutoLogin(true); + LicenseKeyChecker checker = mock(LicenseKeyChecker.class); + when(checker.getPremiumLicenseEnabledResult()) + .thenReturn(KeygenLicenseVerifier.License.SERVER); + + EEAppConfig cfg = new EEAppConfig(props, checker); + + assertThat(cfg.ssoAutoLogin()).isTrue(); + } + + @Test + void ssoAutoLogin_enabled_withoutLicense_throwsAtBootTime() { + ApplicationProperties props = new ApplicationProperties(); + props.getPremium().getProFeatures().setSsoAutoLogin(true); + LicenseKeyChecker checker = mock(LicenseKeyChecker.class); + // Real LicenseKeyChecker.requireProOrEnterprise throws on NORMAL; mock that behavior here. + org.mockito.Mockito.doThrow( + new IllegalStateException( + "premium.proFeatures.ssoAutoLogin=true requires a Pro or Enterprise license")) + .when(checker) + .requireProOrEnterprise("premium.proFeatures.ssoAutoLogin=true"); + + EEAppConfig cfg = new EEAppConfig(props, checker); + + assertThatThrownBy(cfg::ssoAutoLogin) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining( + "premium.proFeatures.ssoAutoLogin=true requires a Pro or Enterprise license"); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/configuration/ee/LicenseKeyCheckerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/configuration/ee/LicenseKeyCheckerTest.java index b30b7fdbd..ae4fa2162 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/security/configuration/ee/LicenseKeyCheckerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/configuration/ee/LicenseKeyCheckerTest.java @@ -1,5 +1,7 @@ package stirling.software.proprietary.security.configuration.ee; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -86,4 +88,43 @@ class LicenseKeyCheckerTest { assertEquals(License.NORMAL, checker.getPremiumLicenseEnabledResult()); verifyNoInteractions(verifier); } + + // ----- requireProOrEnterprise: shared boot-time gate for premium features ----- + + @Test + void requireProOrEnterprise_normalLicense_throwsWithFeatureName() { + LicenseKeyChecker checker = checkerWithLicense(License.NORMAL); + assertThatThrownBy(() -> checker.requireProOrEnterprise("storage.provider=s3")) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("storage.provider=s3 requires a Pro or Enterprise license"); + } + + @Test + void requireProOrEnterprise_serverLicense_passes() { + LicenseKeyChecker checker = checkerWithLicense(License.SERVER); + assertThatCode(() -> checker.requireProOrEnterprise("any.feature=true")) + .doesNotThrowAnyException(); + } + + @Test + void requireProOrEnterprise_enterpriseLicense_passes() { + LicenseKeyChecker checker = checkerWithLicense(License.ENTERPRISE); + assertThatCode(() -> checker.requireProOrEnterprise("any.feature=true")) + .doesNotThrowAnyException(); + } + + private LicenseKeyChecker checkerWithLicense(License level) { + ApplicationProperties props = new ApplicationProperties(); + if (level == License.NORMAL) { + props.getPremium().setEnabled(false); + } else { + props.getPremium().setEnabled(true); + props.getPremium().setKey("any"); + when(verifier.verifyLicense("any")).thenReturn(level); + } + LicenseKeyChecker checker = + new LicenseKeyChecker(verifier, props, userLicenseSettingsService); + checker.init(); + return checker; + } } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/storage/config/ClusterStorageGateTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/storage/config/ClusterStorageGateTest.java new file mode 100644 index 000000000..ee7dc1da3 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/storage/config/ClusterStorageGateTest.java @@ -0,0 +1,250 @@ +package stirling.software.proprietary.storage.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; + +import org.junit.jupiter.api.Test; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.security.configuration.ee.KeygenLicenseVerifier.License; +import stirling.software.proprietary.security.configuration.ee.LicenseKeyChecker; + +class ClusterStorageGateTest { + + @Test + void clusterDisabled_localStorage_passes() { + ClusterStorageGate gate = newGate(false, true, "local", "local"); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + @Test + void clusterDisabled_s3Storage_passes() { + ClusterStorageGate gate = newGate(false, true, "s3", "local"); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + @Test + void clusterEnabled_storageDisabled_butArtifactStoreLocal_fails() { + ClusterStorageGate gate = newGate(true, false, "local", "local"); + assertThatThrownBy(gate::validate) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("cluster.artifactStore=local"); + } + + @Test + void clusterEnabled_storageDisabled_artifactStoreS3_passes() { + ClusterStorageGate gate = newGate(true, false, "local", "s3"); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + @Test + void clusterEnabled_localStorage_fails() { + ClusterStorageGate gate = newGate(true, true, "local", "s3"); + assertThatThrownBy(gate::validate) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("storage.provider=local") + .hasMessageContaining("storage.provider=s3") + .hasMessageContaining("storage.provider=database"); + } + + @Test + void clusterEnabled_localStorage_caseInsensitive_fails() { + ClusterStorageGate gate = newGate(true, true, "LOCAL", "s3"); + assertThatThrownBy(gate::validate).isInstanceOf(IllegalStateException.class); + } + + @Test + void clusterEnabled_nullProvider_treatedAsLocal_fails() { + ClusterStorageGate gate = newGate(true, true, null, "s3"); + assertThatThrownBy(gate::validate) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("storage.provider=local"); + } + + @Test + void clusterEnabled_s3Storage_andArtifactStoreS3_passes() { + ClusterStorageGate gate = newGate(true, true, "s3", "s3"); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + @Test + void clusterEnabled_databaseStorage_andArtifactStoreS3_passes() { + ClusterStorageGate gate = newGate(true, true, "database", "s3"); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + @Test + void clusterEnabled_s3Storage_butLocalArtifactStore_fails() { + ClusterStorageGate gate = newGate(true, true, "s3", "local"); + assertThatThrownBy(gate::validate) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("cluster.artifactStore=local"); + } + + @Test + void clusterEnabled_localArtifactStore_caseInsensitive_fails() { + ClusterStorageGate gate = newGate(true, true, "s3", "LOCAL"); + assertThatThrownBy(gate::validate).isInstanceOf(IllegalStateException.class); + } + + @Test + void clusterEnabled_nullArtifactStore_treatedAsLocal_fails() { + ClusterStorageGate gate = newGate(true, true, "s3", null); + assertThatThrownBy(gate::validate) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("cluster.artifactStore=local"); + } + + @Test + void clusterEnabled_nullStorageObject_passesProviderCheck_butArtifactStoreStillEvaluated() { + ApplicationProperties props = new ApplicationProperties(); + props.setStorage(null); + ClusterStorageGate gate = new ClusterStorageGate(props, mockLicenseChecker(License.SERVER)); + setClusterEnabled(gate, true); + setClusterArtifactStore(gate, "s3"); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + // ----- License gating for premium storage backends ----- + + @Test + void storageProviderS3_withoutProLicense_throws() { + ClusterStorageGate gate = newGate(false, true, "s3", "local", License.NORMAL); + assertThatThrownBy(gate::validate) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("storage.provider=s3 requires a Pro or Enterprise license"); + } + + @Test + void storageProviderDatabase_withoutProLicense_throws() { + ClusterStorageGate gate = newGate(false, true, "database", "local", License.NORMAL); + assertThatThrownBy(gate::validate) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining( + "storage.provider=database requires a Pro or Enterprise license"); + } + + @Test + void storageProviderS3_withServerLicense_passes() { + ClusterStorageGate gate = newGate(false, true, "s3", "local", License.SERVER); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + @Test + void storageProviderS3_withEnterpriseLicense_passes() { + ClusterStorageGate gate = newGate(false, true, "s3", "local", License.ENTERPRISE); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + @Test + void storageProviderDatabase_withServerLicense_passes() { + ClusterStorageGate gate = newGate(false, true, "database", "local", License.SERVER); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + @Test + void clusterArtifactStoreS3_withoutProLicense_throws() { + ClusterStorageGate gate = newGate(false, false, "local", "s3", License.NORMAL); + assertThatThrownBy(gate::validate) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining( + "cluster.artifactStore=s3 requires a Pro or Enterprise license"); + } + + @Test + void clusterArtifactStoreS3_withServerLicense_passes() { + ClusterStorageGate gate = newGate(false, false, "local", "s3", License.SERVER); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + @Test + void localOnly_normalLicense_passes_licenseNotChecked() { + ClusterStorageGate gate = newGate(false, true, "local", "local", License.NORMAL); + assertThatCode(gate::validate).doesNotThrowAnyException(); + } + + @Test + void storageDisabled_butArtifactStoreS3_withoutLicense_stillThrows() { + ClusterStorageGate gate = newGate(false, false, "local", "s3", License.NORMAL); + assertThatThrownBy(gate::validate) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("cluster.artifactStore=s3"); + } + + private static ClusterStorageGate newGate( + boolean clusterEnabled, + boolean storageEnabled, + String provider, + String clusterArtifactStore) { + // Default to a SERVER license so existing tests (which assert clustering / artifact-store + // rules independently of license) continue to pass. License-specific tests below build + // gates with explicit license tiers. + return newGate( + clusterEnabled, storageEnabled, provider, clusterArtifactStore, License.SERVER); + } + + private static ClusterStorageGate newGate( + boolean clusterEnabled, + boolean storageEnabled, + String provider, + String clusterArtifactStore, + License license) { + ApplicationProperties props = new ApplicationProperties(); + ApplicationProperties.Storage storage = new ApplicationProperties.Storage(); + storage.setEnabled(storageEnabled); + storage.setProvider(provider); + props.setStorage(storage); + LicenseKeyChecker checker = mockLicenseChecker(license); + ClusterStorageGate gate = new ClusterStorageGate(props, checker); + setClusterEnabled(gate, clusterEnabled); + setClusterArtifactStore(gate, clusterArtifactStore); + return gate; + } + + private static LicenseKeyChecker mockLicenseChecker(License license) { + LicenseKeyChecker checker = mock(LicenseKeyChecker.class); + when(checker.getPremiumLicenseEnabledResult()).thenReturn(license); + if (license == License.SERVER || license == License.ENTERPRISE) { + doNothing().when(checker).requireProOrEnterprise(anyString()); + } else { + // Mirror real LicenseKeyChecker.requireProOrEnterprise so message assertions match. + org.mockito.Mockito.doAnswer( + inv -> { + throw new IllegalStateException( + inv.getArgument(0) + + " requires a Pro or Enterprise license"); + }) + .when(checker) + .requireProOrEnterprise(anyString()); + } + return checker; + } + + private static void setClusterEnabled(ClusterStorageGate gate, boolean enabled) { + try { + Field f = ClusterStorageGate.class.getDeclaredField("clusterEnabled"); + f.setAccessible(true); + f.setBoolean(gate, enabled); + assertThat(f.getBoolean(gate)).isEqualTo(enabled); + } catch (ReflectiveOperationException e) { + throw new AssertionError("Failed to set clusterEnabled via reflection", e); + } + } + + private static void setClusterArtifactStore(ClusterStorageGate gate, String value) { + try { + Field f = ClusterStorageGate.class.getDeclaredField("clusterArtifactStore"); + f.setAccessible(true); + f.set(gate, value); + } catch (ReflectiveOperationException e) { + throw new AssertionError("Failed to set clusterArtifactStore via reflection", e); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/storage/config/StorageProviderConfigTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/storage/config/StorageProviderConfigTest.java new file mode 100644 index 000000000..ddd67db02 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/storage/config/StorageProviderConfigTest.java @@ -0,0 +1,116 @@ +package stirling.software.proprietary.storage.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.security.configuration.ee.KeygenLicenseVerifier.License; +import stirling.software.proprietary.security.configuration.ee.LicenseKeyChecker; +import stirling.software.proprietary.storage.provider.LocalStorageProvider; +import stirling.software.proprietary.storage.provider.StorageProvider; +import stirling.software.proprietary.storage.repository.StoredFileBlobRepository; + +/** + * Verifies the Pro/Enterprise license gate on the S3 storage backend without touching real S3 + * clients (and without needing Docker). Provider-specific construction is delegated to the existing + * provider tests. + */ +class StorageProviderConfigTest { + + @Test + void provider_local_normalLicense_buildsLocalProviderWithoutLicenseCheck() { + StorageProviderConfig cfg = newConfig("local", License.NORMAL); + + StorageProvider provider = cfg.storageProvider(); + assertThat(provider).isInstanceOf(LocalStorageProvider.class); + } + + @Test + void provider_s3_normalLicense_throwsBeforeBuildingClient() { + StorageProviderConfig cfg = newConfig("s3", License.NORMAL); + + // License check must throw BEFORE S3Clients.build tries to validate endpoint / bucket. + // Otherwise an empty config would surface as a confusing "bucket must be set" error. + assertThatThrownBy(cfg::storageProvider) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("storage.provider=s3 requires a Pro or Enterprise license"); + } + + @Test + void provider_database_normalLicense_throws() { + StorageProviderConfig cfg = newConfig("database", License.NORMAL); + + assertThatThrownBy(cfg::storageProvider) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining( + "storage.provider=database requires a Pro or Enterprise license"); + } + + @Test + void provider_database_serverLicense_buildsDatabaseProvider() { + StorageProviderConfig cfg = newConfig("database", License.SERVER); + assertThatCode(cfg::storageProvider).doesNotThrowAnyException(); + } + + @Test + void provider_s3_serverLicense_passesLicenseCheck_thenFailsOnEmptyConfig() { + StorageProviderConfig cfg = newConfig("s3", License.SERVER); + + // Valid license, but no bucket/endpoint configured - so we expect a CONFIG error, + // not a license error. The error message must not mention the license. + assertThatThrownBy(cfg::storageProvider) + .isInstanceOf(IllegalStateException.class) + .hasMessageNotContaining("Pro or Enterprise license"); + } + + @Test + void provider_s3_enterpriseLicense_passesLicenseCheck_thenFailsOnEmptyConfig() { + StorageProviderConfig cfg = newConfig("s3", License.ENTERPRISE); + + assertThatThrownBy(cfg::storageProvider) + .isInstanceOf(IllegalStateException.class) + .hasMessageNotContaining("Pro or Enterprise license"); + } + + @Test + void provider_unknown_normalLicense_throwsUnsupportedProvider_notLicense() { + StorageProviderConfig cfg = newConfig("magic", License.NORMAL); + + assertThatThrownBy(cfg::storageProvider) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Storage provider not supported: magic") + .hasMessageNotContaining("license"); + } + + private static StorageProviderConfig newConfig(String provider, License license) { + ApplicationProperties props = new ApplicationProperties(); + props.getStorage().setProvider(provider); + props.getStorage() + .setEnabled(false); // local-fallback path skips dir creation when disabled + StoredFileBlobRepository repo = mock(StoredFileBlobRepository.class); + LicenseKeyChecker checker = mock(LicenseKeyChecker.class); + when(checker.getPremiumLicenseEnabledResult()).thenReturn(license); + if (license == License.SERVER || license == License.ENTERPRISE) { + doNothing().when(checker).requireProOrEnterprise(anyString()); + } else { + // Mirror real LicenseKeyChecker.requireProOrEnterprise so message assertions match. + doAnswer( + inv -> { + throw new IllegalStateException( + inv.getArgument(0) + + " requires a Pro or Enterprise license"); + }) + .when(checker) + .requireProOrEnterprise(anyString()); + } + return new StorageProviderConfig(props, repo, checker); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/storage/controller/FileStorageControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/storage/controller/FileStorageControllerTest.java new file mode 100644 index 000000000..79033abc2 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/storage/controller/FileStorageControllerTest.java @@ -0,0 +1,130 @@ +package stirling.software.proprietary.storage.controller; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.redirectedUrl; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.net.URI; +import java.time.Duration; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.storage.model.StoredFile; +import stirling.software.proprietary.storage.provider.StorageProvider; +import stirling.software.proprietary.storage.service.FileStorageService; + +@ExtendWith(MockitoExtension.class) +class FileStorageControllerTest { + + private static final String SIGNED_URL = + "https://test-bucket.s3.example.com/signed-blob?X-Amz-Signature=abc"; + + @Mock private FileStorageService fileStorageService; + @Mock private StorageProvider storageProvider; + + private MockMvc mockMvc; + + @BeforeEach + void setUp() { + FileStorageController controller = + new FileStorageController(fileStorageService, storageProvider); + mockMvc = MockMvcBuilders.standaloneSetup(controller).build(); + } + + @Test + void downloadFile_whenProviderReturnsSignedUrl_returns302RedirectWithoutSessionCredentials() + throws Exception { + StoredFile file = newStoredFile(); + + when(fileStorageService.requireAuthenticatedUser()).thenReturn(file.getOwner()); + when(fileStorageService.getAccessibleFile(file.getOwner(), 77L)).thenReturn(file); + when(storageProvider.signedDownloadUrl( + eq("11/abc-doc.pdf"), any(Duration.class), anyBoolean(), anyString())) + .thenReturn(Optional.of(URI.create(SIGNED_URL))); + + MvcResult result = + mockMvc.perform(get("/api/v1/storage/files/{fileId}/download", 77L)) + .andExpect(status().is(HttpStatus.FOUND.value())) + .andExpect(header().string(HttpHeaders.LOCATION, SIGNED_URL)) + .andExpect(redirectedUrl(SIGNED_URL)) + .andReturn(); + + // Regression fence: signed URLs delegate auth to the URL itself, so the redirect + // response must NOT carry any session credentials forward. + assertThat(result.getResponse().getHeader(HttpHeaders.AUTHORIZATION)).isNull(); + assertThat(result.getResponse().getHeader(HttpHeaders.COOKIE)).isNull(); + assertThat(result.getResponse().getHeader(HttpHeaders.SET_COOKIE)).isNull(); + } + + @Test + void downloadFile_inlineFalse_forwardsAttachmentDispositionToSignedUrl() throws Exception { + StoredFile file = newStoredFile(); + + when(fileStorageService.requireAuthenticatedUser()).thenReturn(file.getOwner()); + when(fileStorageService.getAccessibleFile(file.getOwner(), 77L)).thenReturn(file); + when(storageProvider.signedDownloadUrl( + eq("11/abc-doc.pdf"), any(Duration.class), eq(false), eq("doc.pdf"))) + .thenReturn(Optional.of(URI.create(SIGNED_URL))); + + mockMvc.perform(get("/api/v1/storage/files/{fileId}/download", 77L)) + .andExpect(status().is(HttpStatus.FOUND.value())) + .andExpect(header().string(HttpHeaders.LOCATION, SIGNED_URL)); + + verify(storageProvider) + .signedDownloadUrl( + eq("11/abc-doc.pdf"), any(Duration.class), eq(false), eq("doc.pdf")); + } + + @Test + void downloadFile_inlineTrue_forwardsInlineDispositionToSignedUrl() throws Exception { + StoredFile file = newStoredFile(); + + when(fileStorageService.requireAuthenticatedUser()).thenReturn(file.getOwner()); + when(fileStorageService.getAccessibleFile(file.getOwner(), 77L)).thenReturn(file); + when(storageProvider.signedDownloadUrl( + eq("11/abc-doc.pdf"), any(Duration.class), eq(true), eq("doc.pdf"))) + .thenReturn(Optional.of(URI.create(SIGNED_URL))); + + mockMvc.perform(get("/api/v1/storage/files/{fileId}/download", 77L).param("inline", "true")) + .andExpect(status().is(HttpStatus.FOUND.value())) + .andExpect(header().string(HttpHeaders.LOCATION, SIGNED_URL)); + + verify(storageProvider) + .signedDownloadUrl( + eq("11/abc-doc.pdf"), any(Duration.class), eq(true), eq("doc.pdf")); + } + + private static StoredFile newStoredFile() { + User user = new User(); + user.setId(11L); + user.setUsername("alice"); + + StoredFile file = new StoredFile(); + file.setId(77L); + file.setOwner(user); + file.setOriginalFilename("doc.pdf"); + file.setContentType("application/pdf"); + file.setSizeBytes(123L); + file.setStorageKey("11/abc-doc.pdf"); + return file; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/storage/provider/S3StorageProviderTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/storage/provider/S3StorageProviderTest.java new file mode 100644 index 000000000..e3fde2cf4 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/storage/provider/S3StorageProviderTest.java @@ -0,0 +1,282 @@ +package stirling.software.proprietary.storage.provider; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Optional; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.core.io.Resource; +import org.springframework.mock.web.MockMultipartFile; +import org.testcontainers.containers.MinIOContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import stirling.software.proprietary.security.model.User; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; + +@Testcontainers(disabledWithoutDocker = true) +class S3StorageProviderTest { + + private static final String BUCKET = "stirling-test-bucket"; + private static final String ACCESS_KEY = "minioadmin"; + private static final String SECRET_KEY = "minioadmin"; + + @Container + static MinIOContainer minio = + new MinIOContainer("minio/minio:latest") + .withUserName(ACCESS_KEY) + .withPassword(SECRET_KEY); + + private static S3Client s3Client; + private static S3Presigner s3Presigner; + private static S3StorageProvider provider; + + @BeforeAll + static void setUp() { + URI endpoint = URI.create(minio.getS3URL()); + AwsBasicCredentials creds = AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY); + S3Configuration s3Config = S3Configuration.builder().pathStyleAccessEnabled(true).build(); + + s3Client = + S3Client.builder() + .endpointOverride(endpoint) + .httpClient(UrlConnectionHttpClient.create()) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create(creds)) + .serviceConfiguration(s3Config) + .build(); + + s3Presigner = + S3Presigner.builder() + .endpointOverride(endpoint) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create(creds)) + .serviceConfiguration(s3Config) + .build(); + + s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build()); + provider = new S3StorageProvider(s3Client, s3Presigner, BUCKET); + } + + @AfterAll + static void tearDown() { + if (provider != null) { + provider.close(); + } + } + + @Test + void blankBucket_constructorRejects() { + assertThatThrownBy(() -> new S3StorageProvider(s3Client, s3Presigner, "")) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new S3StorageProvider(s3Client, s3Presigner, null)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void store_thenLoad_roundTripsContent() throws Exception { + User owner = new User(); + owner.setId(42L); + byte[] content = "hello s3 round trip".getBytes(StandardCharsets.UTF_8); + MockMultipartFile file = + new MockMultipartFile("file", "sample.pdf", "application/pdf", content); + + StoredObject stored = provider.store(owner, file); + + // Key is intentionally opaque ({ownerId}/{uuid}) - the filename is preserved on + // StoredObject.originalFilename for display, never in the S3 key, so vendors that + // restrict key charset (e.g. Supabase: ASCII only) accept any filename. + assertThat(stored.getStorageKey()) + .matches( + "42/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"); + assertThat(stored.getStorageKey()).doesNotContain("sample.pdf"); + assertThat(stored.getOriginalFilename()).isEqualTo("sample.pdf"); + assertThat(stored.getContentType()).isEqualTo("application/pdf"); + assertThat(stored.getSizeBytes()).isEqualTo(content.length); + + Resource loaded = provider.load(stored.getStorageKey()); + try (InputStream in = loaded.getInputStream()) { + assertThat(in.readAllBytes()).isEqualTo(content); + } + } + + @Test + void load_unknownKey_throwsIOException() { + assertThatThrownBy(() -> provider.load("does/not/exist.txt")) + .isInstanceOf(IOException.class); + } + + @Test + void store_unicodeFilename_yieldsAsciiOnlyKey_andPreservesOriginalName() throws Exception { + // Regression: Supabase Storage rejects S3 keys containing non-ASCII chars (400 + // Invalid key). Locking in that the storage key never embeds the filename so any + // unicode display name still uploads successfully. + User owner = new User(); + owner.setId(99L); + String unicodeName = "résumé-日本語-é.pdf"; + byte[] payload = "u".getBytes(StandardCharsets.UTF_8); + MockMultipartFile file = + new MockMultipartFile("file", unicodeName, "application/pdf", payload); + + StoredObject stored = provider.store(owner, file); + + assertThat(stored.getStorageKey()) + .matches( + "99/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"); + assertThat(stored.getOriginalFilename()).isEqualTo(unicodeName); + try (InputStream in = provider.load(stored.getStorageKey()).getInputStream()) { + assertThat(in.readAllBytes()).isEqualTo(payload); + } + } + + @Test + void delete_removesObject() throws Exception { + User owner = new User(); + owner.setId(7L); + MockMultipartFile file = + new MockMultipartFile( + "file", "todelete.bin", "application/octet-stream", new byte[] {1, 2, 3}); + + StoredObject stored = provider.store(owner, file); + provider.delete(stored.getStorageKey()); + + assertThatThrownBy(() -> provider.load(stored.getStorageKey())) + .isInstanceOf(IOException.class); + } + + @Test + void delete_unknownKey_isNoOp() { + assertThat(catchIOException(() -> provider.delete("never-existed"))).isNull(); + } + + @Test + void signedDownloadUrl_returnsWorkingPresignedGet() throws Exception { + User owner = new User(); + owner.setId(99L); + byte[] content = "presigned payload".getBytes(StandardCharsets.UTF_8); + StoredObject stored = + provider.store( + owner, new MockMultipartFile("file", "presign.txt", "text/plain", content)); + + Optional signed = + provider.signedDownloadUrl(stored.getStorageKey(), Duration.ofMinutes(2)); + + assertThat(signed).isPresent(); + URI uri = signed.get(); + assertThat(uri.getScheme()).isIn("http", "https"); + assertThat(uri.getRawQuery()).contains("X-Amz-Signature"); + + HttpURLConnection conn = (HttpURLConnection) new URL(uri.toString()).openConnection(); + try { + assertThat(conn.getResponseCode()).isEqualTo(200); + try (InputStream in = conn.getInputStream()) { + assertThat(in.readAllBytes()).isEqualTo(content); + } + } finally { + conn.disconnect(); + } + } + + @Test + void signedDownloadUrl_nullKey_returnsEmpty() throws Exception { + assertThat(provider.signedDownloadUrl(null, Duration.ofMinutes(1))).isEmpty(); + assertThat(provider.signedDownloadUrl(" ", Duration.ofMinutes(1))).isEmpty(); + } + + @Test + void signedDownloadUrl_nullOrZeroTtl_appliesDefault() throws Exception { + User owner = new User(); + owner.setId(3L); + StoredObject stored = + provider.store( + owner, + new MockMultipartFile( + "file", + "ttl.txt", + "text/plain", + "x".getBytes(StandardCharsets.UTF_8))); + + assertThat(provider.signedDownloadUrl(stored.getStorageKey(), null)).isPresent(); + assertThat(provider.signedDownloadUrl(stored.getStorageKey(), Duration.ZERO)).isPresent(); + assertThat(provider.signedDownloadUrl(stored.getStorageKey(), Duration.ofSeconds(-5))) + .isPresent(); + } + + @Test + void signedDownloadUrl_inlineFlagEncodesResponseContentDispositionInQuery() throws Exception { + User owner = new User(); + owner.setId(55L); + StoredObject stored = + provider.store( + owner, + new MockMultipartFile( + "file", + "stored-name.pdf", + "application/pdf", + "payload".getBytes(StandardCharsets.UTF_8))); + + URI attached = + provider.signedDownloadUrl( + stored.getStorageKey(), Duration.ofMinutes(2), false, "report.pdf") + .orElseThrow(); + String attachedQuery = + java.net.URLDecoder.decode(attached.getRawQuery(), StandardCharsets.UTF_8); + assertThat(attachedQuery) + .contains("response-content-disposition=attachment; filename=\"report.pdf\""); + + URI inline = + provider.signedDownloadUrl( + stored.getStorageKey(), Duration.ofMinutes(2), true, "report.pdf") + .orElseThrow(); + String inlineQuery = + java.net.URLDecoder.decode(inline.getRawQuery(), StandardCharsets.UTF_8); + assertThat(inlineQuery) + .contains("response-content-disposition=inline; filename=\"report.pdf\""); + + URI bare = + provider.signedDownloadUrl( + stored.getStorageKey(), Duration.ofMinutes(2), false, null) + .orElseThrow(); + assertThat(bare.getRawQuery()).doesNotContain("response-content-disposition"); + } + + @Test + void buildContentDisposition_escapesQuotesAndStripsControlChars() { + assertThat(S3StorageProvider.buildContentDisposition(true, "ev\"il\r\nname.pdf")) + .isEqualTo("inline; filename=\"ev\\\"ilname.pdf\""); + assertThat(S3StorageProvider.buildContentDisposition(false, null)).isNull(); + assertThat(S3StorageProvider.buildContentDisposition(false, " ")).isNull(); + } + + private static IOException catchIOException(IOAction action) { + try { + action.run(); + return null; + } catch (IOException e) { + return e; + } + } + + @FunctionalInterface + private interface IOAction { + void run() throws IOException; + } +}