diff --git a/.github/config/.files.yaml b/.github/config/.files.yaml index 8066cc362..5343246c2 100644 --- a/.github/config/.files.yaml +++ b/.github/config/.files.yaml @@ -38,6 +38,8 @@ project: &project - frontend/** - docker/** - scripts/RestartHelper.java + - scripts/db-migration/** + - .github/workflows/db-migration-test.yml frontend: &frontend - frontend/** diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 05029d2aa..ac47fe134 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -68,6 +68,18 @@ jobs: uses: ./.github/workflows/backend-build.yml secrets: inherit + db-migration-test: + # Boots the current bootJar against H2 fixtures captured from past + # releases (v2.0.0 / v2.5.0 / v2.10.0) and verifies admin login still + # works after Hibernate's ddl-auto=update migrates the schema. Gated on + # the `project` filter so doc-only PRs skip this ~5-minute job. + if: needs.files-changed.outputs.project == 'true' + needs: [files-changed] + permissions: + contents: read + uses: ./.github/workflows/db-migration-test.yml + secrets: inherit + check-generateOpenApiDocs: if: needs.files-changed.outputs.openapi == 'true' needs: [files-changed] @@ -184,6 +196,7 @@ jobs: needs: - files-changed - build + - db-migration-test - check-generateOpenApiDocs - frontend-validation - playwright-e2e @@ -208,6 +221,7 @@ jobs: RESULTS: | files-changed=${{ needs.files-changed.result }} build=${{ needs.build.result }} + db-migration-test=${{ needs.db-migration-test.result }} check-generateOpenApiDocs=${{ needs.check-generateOpenApiDocs.result }} frontend-validation=${{ needs.frontend-validation.result }} playwright-e2e=${{ needs.playwright-e2e.result }} diff --git a/.github/workflows/db-migration-test.yml b/.github/workflows/db-migration-test.yml new file mode 100644 index 000000000..2a386d1a5 --- /dev/null +++ b/.github/workflows/db-migration-test.yml @@ -0,0 +1,93 @@ +name: DB migration smoke test + +# Boots the current Stirling-PDF JAR against H2 fixtures captured from past +# releases (v2.0.0 / v2.5.0 / v2.10.0) and verifies admin login still works. +# Catches schema changes that would break existing user databases under +# Hibernate's `ddl-auto=update` upgrade path. + +on: + workflow_call: + +permissions: + contents: read + +jobs: + pick: + uses: ./.github/workflows/_runner-pick.yml + + migration-test: + needs: pick + runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-8' }} + timeout-minutes: 30 + env: + DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} + steps: + - name: Harden Runner + uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + with: + egress-policy: audit + + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up JDK 25 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: 25 + distribution: temurin + + - name: Cache Gradle dependency artifacts + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: | + ~/.gradle/wrapper + ~/.gradle/caches/modules-2/files-2.1 + ~/.gradle/caches/modules-2/metadata-2.* + key: gradle-deps-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }} + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0 + with: + gradle-version: 9.3.1 + cache-disabled: true + + # No `-PnoSpotless` here yet because the upstream cache layer matches the + # backend build's; reuse keeps cold-cache cost identical. + - name: Build Stirling-PDF JAR + env: + MAVEN_USER: ${{ secrets.MAVEN_USER }} + MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} + MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }} + run: ./gradlew :stirling-pdf:bootJar -PnoSpotless --no-daemon + + - name: Locate built JAR + id: jar + run: | + jar=$(find app/core/build/libs -maxdepth 1 -name 'Stirling-PDF*.jar' -o -name 'stirling-pdf*.jar' 2>/dev/null \ + | grep -vE '(-plain|-sources)\.jar$' | head -n 1) + if [[ -z "$jar" ]]; then + echo "::error::No JAR under app/core/build/libs" + ls -lah app/core/build/libs || true + exit 1 + fi + # Absolute path - the migration script pushd's into a temp workdir + # before invoking java, which would dangle a relative path. + jar=$(realpath "$jar") + echo "path=$jar" >> "$GITHUB_OUTPUT" + echo "Built JAR: $jar" + + - name: Run migration smoke test + env: + STIRLING_JAR: ${{ steps.jar.outputs.path }} + run: bash scripts/db-migration/run-migration-test.sh + + - name: Upload app logs on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: db-migration-app-logs + # Path matches the preserved workdir in run-migration-test.sh - + # only failing fixtures leave a directory behind. + path: /tmp/stirling-migration-failed-*/app.log + retention-days: 7 + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index c67ec629c..f4558af4c 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,10 @@ customFiles/ configs/ watchedFolders/ clientWebUI/ +# Scratch dir used by local fixture-regeneration runs (see +# app/proprietary/src/test/resources/db-migration-fixtures/README.md). +# Holds downloaded JARs and disposable workdirs. Never committed. +.alpha-local/ !cucumber/ !cucumber/exampleFiles/ !cucumber/exampleFiles/example_html.zip diff --git a/app/proprietary/.gitignore b/app/proprietary/.gitignore index 5156eb74a..1a5610dc1 100644 --- a/app/proprietary/.gitignore +++ b/app/proprietary/.gitignore @@ -123,6 +123,8 @@ SwaggerDoc.json *.tar.gz *.rar *.db +# Whitelist the H2 fixtures that feed the version-migration CI smoke test. +!src/test/resources/db-migration-fixtures/*.mv.db /build /app/proprietary/build/ diff --git a/app/proprietary/src/test/resources/db-migration-fixtures/README.md b/app/proprietary/src/test/resources/db-migration-fixtures/README.md new file mode 100644 index 000000000..25450cf7f --- /dev/null +++ b/app/proprietary/src/test/resources/db-migration-fixtures/README.md @@ -0,0 +1,110 @@ +# DB migration test fixtures + +These `.mv.db` files are H2 databases captured from past Stirling-PDF releases. +They feed the CI smoke test that verifies a fresh build can still boot and +authenticate against a database created by an older version. + +| File | Source release | Tables | Notes | +|---|---|---|---| +| `stirling-pdf-v2.0.0.mv.db` | [v2.0.0](https://github.com/Stirling-Tools/Stirling-PDF/releases/tag/v2.0.0) | users, authorities, teams, sessions, audit_events, persistent_logins, invite_tokens, user_license_settings, user_settings | Pre-storage/workflow schema. | +| `stirling-pdf-v2.5.0.mv.db` | [v2.5.0](https://github.com/Stirling-Tools/Stirling-PDF/releases/tag/v2.5.0) | same as v2.0.0 | Schema unchanged from v2.0.0; intentionally kept as a separate fixture to exercise the "skip every other minor" upgrade path. | +| `stirling-pdf-v2.10.0.mv.db` | [v2.10.0](https://github.com/Stirling-Tools/Stirling-PDF/releases/tag/v2.10.0) | v2.5.0 tables + file_shares, file_share_accesses, stored_files, stored_file_blobs, storage_cleanup_entries, user_server_certificates, workflow_sessions, workflow_participants, participant_notifications | Adds the file-sharing and workflow signing schema. | + +All three were generated against H2 `2.3.232` and use the same on-disk file +format, so the runtime driver can open any of them without conversion. + +## What's in each fixture + +* `admin` user with the default password `stirling` (BCrypt `$2a$10$...`). +* The internal API user `STIRLING-PDF-BACKEND-API-USER`. +* `ROLE_ADMIN` authority row for the admin user. +* `Default` and `Internal` teams. +* `user_license_settings` row (singleton). + +`audit_events`, `sessions`, and `user_settings` are empty in the OSS-flavored +fixtures: those tables are written only on Enterprise builds (audit) or +require an HTTP-session-creating flow (sessions / settings) that the OSS form +login no longer exposes. The migration test only depends on the admin user +existing, so leaving these empty is intentional. + +## What the CI test checks + +`.github/workflows/db-migration-test.yml` runs `scripts/db-migration/run-migration-test.sh`, +which for each fixture: + +1. Copies the fixture into `configs/stirling-pdf-DB-2.3.232.mv.db` of a clean + working directory. +2. Boots the current `:stirling-pdf:bootJar` against it on a free port. +3. Waits for Spring to start (no `SchemaManagementException` in the log). +4. POSTs `{"username":"admin","password":"stirling"}` to `/api/v1/auth/login` + and asserts the response is `200 OK`. + +A red CI on this job means a schema change in the PR is not backwards +compatible with an existing user database. Common causes: + +* Adding a non-nullable column without a default. +* Renaming a column (Hibernate's `update` strategy adds the new column and + leaves the old one orphaned with the data still in it). +* Changing a column type in an incompatible way. +* Dropping or renaming a foreign-key target. + +## Regenerating fixtures + +There's no automated regenerator script - fixtures are rare to refresh and the +manual steps are short. For each version you want to capture: + +```bash +# 1. Download the JAR for that release (requires `gh` authenticated against +# github.com/Stirling-Tools/Stirling-PDF). +gh release download v2.10.0 \ + --repo Stirling-Tools/Stirling-PDF \ + --pattern 'Stirling-PDF-with-login.jar' \ + --output /tmp/stirling-v2.10.0.jar + +# 2. Boot the JAR in a clean working directory. DB_CLOSE_ON_EXIT=TRUE is +# the only override that matters - it makes the H2 file flush on JVM exit +# even if you Ctrl-C instead of going through a graceful shutdown. +workdir=$(mktemp -d) +mkdir -p "$workdir/configs" +cd "$workdir" +java -jar /tmp/stirling-v2.10.0.jar \ + --server.port=8089 \ + --spring.datasource.url='jdbc:h2:file:./configs/stirling-pdf-DB-2.3.232;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=TRUE;MODE=PostgreSQL' \ + & + +# 3. Wait until http://localhost:8089/login responds, then log in once to +# materialize whatever rows the app writes on first boot. +curl -sf -X POST -H 'Content-Type: application/json' \ + -d '{"username":"admin","password":"stirling"}' \ + http://localhost:8089/api/v1/auth/login + +# 4. Shut it down (any kill works - DB_CLOSE_ON_EXIT=TRUE handles the flush). +kill -TERM %1 && wait %1 + +# 5. Copy the .mv.db here, renamed for the version. +cp "$workdir/configs/stirling-pdf-DB-2.3.232.mv.db" \ + app/proprietary/src/test/resources/db-migration-fixtures/stirling-pdf-v2.10.0.mv.db +``` + +Requirements: Java 21+ (the historical JARs target Java 17 / 21). + +## Adding a new fixture + +When a new minor release ships, repeat the steps above for the new tag and +add a row to the table at the top of this file. Keep the historical fixtures - +the test gets stronger with each schema generation it covers. + +## Inspecting a fixture by hand + +The H2 driver bundled with the build ships an interactive shell: + +```bash +h2_jar=$(find ~/.gradle/caches/modules-2 -name 'h2-2.3.232.jar' | head -1) +cd app/proprietary/src/test/resources/db-migration-fixtures +java -cp "$h2_jar" org.h2.tools.Shell \ + -url 'jdbc:h2:file:./stirling-pdf-v2.10.0;ACCESS_MODE_DATA=r;MODE=PostgreSQL' \ + -user sa +``` + +`ACCESS_MODE_DATA=r` keeps the inspection read-only so you can't accidentally +mutate a committed fixture. diff --git a/app/proprietary/src/test/resources/db-migration-fixtures/stirling-pdf-v2.0.0.mv.db b/app/proprietary/src/test/resources/db-migration-fixtures/stirling-pdf-v2.0.0.mv.db new file mode 100644 index 000000000..5402eb037 Binary files /dev/null and b/app/proprietary/src/test/resources/db-migration-fixtures/stirling-pdf-v2.0.0.mv.db differ diff --git a/app/proprietary/src/test/resources/db-migration-fixtures/stirling-pdf-v2.10.0.mv.db b/app/proprietary/src/test/resources/db-migration-fixtures/stirling-pdf-v2.10.0.mv.db new file mode 100644 index 000000000..56c004055 Binary files /dev/null and b/app/proprietary/src/test/resources/db-migration-fixtures/stirling-pdf-v2.10.0.mv.db differ diff --git a/app/proprietary/src/test/resources/db-migration-fixtures/stirling-pdf-v2.5.0.mv.db b/app/proprietary/src/test/resources/db-migration-fixtures/stirling-pdf-v2.5.0.mv.db new file mode 100644 index 000000000..1032bf42d Binary files /dev/null and b/app/proprietary/src/test/resources/db-migration-fixtures/stirling-pdf-v2.5.0.mv.db differ diff --git a/scripts/db-migration/run-migration-test.sh b/scripts/db-migration/run-migration-test.sh new file mode 100644 index 000000000..953f58712 --- /dev/null +++ b/scripts/db-migration/run-migration-test.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# +# DB migration smoke test: for each H2 fixture under +# app/proprietary/src/test/resources/db-migration-fixtures/, copy it into a +# fresh working directory, boot the current Stirling-PDF JAR against it, then +# POST /api/v1/auth/login with the fixture's admin credentials. A 200 means +# Hibernate's ddl-auto=update migrated the legacy schema without breaking +# existing data. +# +# Inputs: +# STIRLING_JAR - path to a pre-built Stirling-PDF .jar (defaults to the +# :stirling-pdf:bootJar output) +# FIXTURE_DIR - override the fixture directory (rarely needed) +# +# Exits non-zero on any fixture failure and writes a summary to stderr. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +FIXTURE_DIR="${FIXTURE_DIR:-$REPO_ROOT/app/proprietary/src/test/resources/db-migration-fixtures}" +STIRLING_JAR="${STIRLING_JAR:-}" +ADMIN_USERNAME="${ADMIN_USERNAME:-admin}" +ADMIN_PASSWORD="${ADMIN_PASSWORD:-stirling}" +STARTUP_TIMEOUT_SEC="${STARTUP_TIMEOUT_SEC:-300}" + +log() { printf '[migration-test] %s\n' "$*" >&2; } +fail() { printf '[migration-test][FAIL] %s\n' "$*" >&2; exit 1; } + +find_jar() { + local candidate + if [[ -n "$STIRLING_JAR" ]]; then + [[ -f "$STIRLING_JAR" ]] || fail "STIRLING_JAR='$STIRLING_JAR' not found" + candidate="$STIRLING_JAR" + else + candidate=$(find "$REPO_ROOT/app/core/build/libs" -maxdepth 1 -name 'Stirling-PDF*.jar' -o -name 'stirling-pdf*.jar' 2>/dev/null \ + | grep -vE '(-plain|-sources)\.jar$' | head -n 1 || true) + [[ -n "$candidate" ]] || fail "No JAR under app/core/build/libs - run './gradlew :stirling-pdf:bootJar' first" + fi + # Resolve to an absolute path: test_fixture pushd's into a temp workdir + # before launching java, so a relative path here would dangle. + realpath "$candidate" +} + +free_port() { + python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()' +} + +wait_for_url() { + local url="$1" deadline=$(( $(date +%s) + STARTUP_TIMEOUT_SEC )) + while (( $(date +%s) < deadline )); do + local code + code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 "$url" || true) + # Any HTTP response (200, 302, 401, 403, 404) means Spring is serving. + if [[ "$code" =~ ^[1-5][0-9][0-9]$ ]]; then return 0; fi + sleep 2 + done + return 1 +} + +test_fixture() { + local fixture_path="$1" + local label + label=$(basename "$fixture_path" .mv.db) + log "=== $label ===" + + local jar; jar=$(find_jar) + local workdir; workdir=$(mktemp -d) + local configsdir="$workdir/configs" + mkdir -p "$configsdir" + cp "$fixture_path" "$configsdir/stirling-pdf-DB-2.3.232.mv.db" + + local port; port=$(free_port) + local base_url="http://127.0.0.1:$port" + local log_file="$workdir/app.log" + + log " jar=$jar" + log " workdir=$workdir" + log " port=$port" + + # Force DB_CLOSE_ON_EXIT=TRUE so the H2 file flushes via shutdown hook even + # on a hard kill. show-sql off (output is enormous) but schema-tool INFO + # is on so any migration failure is visible in CI logs. + # + # pushd into the workdir so the JVM's `.` resolves to the isolated + # workdir - InstallationPathConfig reads `./configs/settings.yml` relative + # to cwd, and we want to make sure we hit the fixture's configs/ and not + # whatever happens to live at the runner's working directory. + pushd "$workdir" >/dev/null + java -Xmx1g -jar "$jar" \ + "--server.port=$port" \ + "--spring.datasource.url=jdbc:h2:file:./configs/stirling-pdf-DB-2.3.232;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=TRUE;MODE=PostgreSQL" \ + "--spring.jpa.show-sql=false" \ + "--logging.level.root=WARN" \ + "--logging.level.stirling=INFO" \ + "--logging.level.org.hibernate.tool.schema=INFO" \ + > "$log_file" 2>&1 & + local pid=$! + popd >/dev/null + + local rc=0 + if ! wait_for_url "$base_url/login"; then + log " app did not start within $STARTUP_TIMEOUT_SEC sec; last 60 lines of stdout:" + tail -n 60 "$log_file" >&2 + rc=1 + else + log " app started" + + if grep -E -i 'SchemaManagementException|GenerationTarget encountered exception|liquibase|Flyway.*FAILED' "$log_file" >&2; then + log " Hibernate reported schema migration errors (see log above)" + rc=1 + fi + + if (( rc == 0 )); then + local login_body; login_body=$(printf '{"username":"%s","password":"%s"}' "$ADMIN_USERNAME" "$ADMIN_PASSWORD") + local resp_file="$workdir/login.resp" + local code + code=$(curl -s -o "$resp_file" -w '%{http_code}' \ + --max-time 30 \ + -H 'Content-Type: application/json' \ + -d "$login_body" \ + "$base_url/api/v1/auth/login" || echo "000") + log " POST /api/v1/auth/login -> HTTP $code" + if [[ "$code" != "200" ]]; then + log " response body:" + head -c 500 "$resp_file" >&2 || true; echo >&2 + log " last 40 lines of app log:" + tail -n 40 "$log_file" >&2 + rc=1 + else + # Cheap sanity check on a second endpoint -- proves the app is + # not just stubbing the login response. + local status_code + status_code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$base_url/api/v1/info/status" || echo "000") + log " GET /api/v1/info/status -> HTTP $status_code" + if [[ "$status_code" != "200" ]]; then + log " sanity check failed: /api/v1/info/status returned non-200" + rc=1 + else + log " PASS: $label migrated and admin login succeeded" + fi + fi + fi + fi + + # H2 flushes via DB_CLOSE_ON_EXIT=TRUE so a forced kill is safe. + kill -TERM "$pid" 2>/dev/null || true + local i + for i in $(seq 1 15); do + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + + if (( rc == 0 )); then + rm -rf "$workdir" 2>/dev/null || true + else + # Preserve workdir so CI can upload the app log as an artifact for + # post-mortem. Move it to a stable name so the upload path is fixed. + local preserved + preserved="${MIGRATION_TEST_LOG_DIR:-/tmp}/stirling-migration-failed-$label" + rm -rf "$preserved" + mv "$workdir" "$preserved" 2>/dev/null || true + log " preserved failing workdir at $preserved" + fi + + return $rc +} + +main() { + [[ -d "$FIXTURE_DIR" ]] || fail "Fixture dir not found: $FIXTURE_DIR" + local fixtures + mapfile -t fixtures < <(find "$FIXTURE_DIR" -maxdepth 1 -name '*.mv.db' | sort) + [[ ${#fixtures[@]} -gt 0 ]] || fail "No fixtures under $FIXTURE_DIR" + + local failed=0 + local failed_names=() + for f in "${fixtures[@]}"; do + # set -e would abort the whole run on the first failure; we want to + # report every fixture's status, so guard with ||. + if ! test_fixture "$f"; then + failed=$(( failed + 1 )) + failed_names+=("$(basename "$f" .mv.db)") + fi + done + + if (( failed > 0 )); then + log "${failed}/${#fixtures[@]} fixture(s) failed: ${failed_names[*]}" + exit 1 + fi + log "All ${#fixtures[@]} fixtures migrated cleanly." +} + +main "$@"