mirror of
https://github.com/arsvendg/Stirling-PDF.git
synced 2026-07-16 19:33:11 +02:00
fix(policies): poll runs to completion with progress, soft-retry when queue is full (#6690)
## What & why
Production reports of policy enforcement "hanging" traced to
large/many-page documents: the watermark step's flatten-to-image
(`convertPDFToImage`) on a 500+ page PDF takes minutes, exceeding both
the client poll cap and the backend per-step timeout. This makes the
slow case graceful instead of looking broken, and makes load-shedding
non-fatal.
### Poll runs to completion (no false "hang")
The client poll loop used a flat ~150s cap that was **shorter than the
backend's 300s per-step timeout**, so it abandoned long-but-healthy runs
mid-flight. The budget is now sized to the backend's real worst case —
`stepCount × per-step timeout + grace`, learned from the first status
report — so the client always polls long enough to surface the run's
**actual** terminal state (success or the backend's real error) rather
than a misleading client-side timeout.
### Per-step progress
The activity feed now shows `Enforcing… · step n/m` (from
`currentStep`/`stepCount`), so a slow step shows movement instead of a
dead spinner.
### Soft-retry on queue rejection
Under load the shared `JobQueue` rejects runs ("queue full"), which
previously surfaced as a hard failure needing a manual Retry. The
backend now tags that rejection with a stable `POLICY_QUEUE_FULL`
errorCode; the client treats it as transient backpressure and
**auto-retries the file in place** with exponential backoff (≈4s→64s, ~2
min), shown as a soft "Busy — retrying…" row, falling back to the manual
Retry only once the retry budget is spent.
## Testing
- **Frontend unit tests** (30 pass across the policies suite), including
a new `usePolicyAutoRun.retry.test.tsx` that drives the real controller
orchestration (poll → `POLICY_QUEUE_FULL` → relabel → backoff → in-place
re-dispatch), plus poll-budget, step-progress, and activity-feed relabel
cases.
- **Backend** `PolicyEngineTest` case asserting a queue-rejected run
carries the `POLICY_QUEUE_FULL` code.
- Typecheck clean on all three flavors (proprietary/saas/core); prettier
+ spotless clean.
- Poll-budget + progress + real-error surfacing were also verified live
end-to-end against a 599-page run (survived past the old cap, showed
step progress, reported the backend's real 300s-timeout failure,
recovered after a simulated network drop).
## Not included (follow-ups)
- The underlying flatten-to-image cost itself (bounded-memory/streaming
flatten, revisiting `convertPDFToImage` default and the 300s timeout) —
the real perf fix, deliberately out of scope here.
This commit is contained in:
Reece Browne
@@ -35,17 +35,68 @@ import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext";
|
||||
import { usePolicies } from "@app/hooks/usePolicies";
|
||||
import {
|
||||
dispatchKey,
|
||||
getRun,
|
||||
isDispatched,
|
||||
markDispatched,
|
||||
recordRunStart,
|
||||
removeRun,
|
||||
updateRun,
|
||||
usePolicyRuns,
|
||||
type PolicyRunRecord,
|
||||
} from "@app/components/policies/policyRunStore";
|
||||
|
||||
/** Poll cadence + cap for a single run's status (≈2.5 min worst case). */
|
||||
/** Status poll cadence. */
|
||||
const POLL_MS = 2000;
|
||||
const MAX_POLLS = 75;
|
||||
|
||||
/** The server aborts any single tool step that runs longer than its internal-API
|
||||
* read timeout, then fails the run — so a run can legitimately stay in flight
|
||||
* for up to this long per step. The client must keep polling at least that long,
|
||||
* or it abandons a run the server is still working on (which reads as a hang). */
|
||||
const STEP_TIMEOUT_MS = 300_000;
|
||||
|
||||
/** Slack on top of the per-step budget: queueing before the first step starts and
|
||||
* output handling after the last one finishes. */
|
||||
const POLL_GRACE_MS = 30_000;
|
||||
|
||||
/** Step count assumed before the first status report reveals the real pipeline
|
||||
* length — only governs the budget for those first couple of polls. */
|
||||
const DEFAULT_STEP_COUNT = 4;
|
||||
|
||||
/** errorCode the backend sets when a run is rejected at admission (job queue full under load).
|
||||
* Transient, not a real processing failure — we back off and retry rather than surfacing it. */
|
||||
const POLICY_QUEUE_FULL = "POLICY_QUEUE_FULL";
|
||||
|
||||
/** Auto-retry budget + exponential backoff for a queue-rejected run, to ride out a busy period
|
||||
* before giving up to a manual retry. Delays are BASE × 2^attempt (≈4s, 8s … 64s, ~2min total). */
|
||||
const MAX_QUEUE_RETRIES = 5;
|
||||
const QUEUE_RETRY_BASE_MS = 4000;
|
||||
|
||||
/** Consecutive "run not found" responses before giving up. The run state lives
|
||||
* in memory on the server, so a restart or a second instance behind the load
|
||||
* balancer makes a live run's status return 404 — and it won't come back. We
|
||||
* tolerate a brief blip (e.g. a poll racing a just-dispatched run, or one hop
|
||||
* to an instance that hasn't seen it) then fail, rather than polling forever. */
|
||||
const MAX_NOT_FOUND = 3;
|
||||
|
||||
/** A 404 from the run-status endpoint, across the web (axios) and desktop
|
||||
* (tauri http client → {@code code: "ERR_NOT_FOUND"}) builds. */
|
||||
function isRunNotFound(err: unknown): boolean {
|
||||
const e = err as
|
||||
| { code?: string; status?: number; response?: { status?: number } }
|
||||
| null
|
||||
| undefined;
|
||||
return (
|
||||
e?.code === "ERR_NOT_FOUND" ||
|
||||
e?.status === 404 ||
|
||||
e?.response?.status === 404
|
||||
);
|
||||
}
|
||||
|
||||
/** Mark a run terminal-failed so it stops being polled (and re-polled on reload)
|
||||
* and the activity feed offers Retry, instead of the file enforcing forever. */
|
||||
function failRun(runId: string, message: string): void {
|
||||
updateRun(runId, { status: "FAILED", error: message, errorCode: null });
|
||||
}
|
||||
|
||||
/** How long to wait for an upload's bytes to land in IndexedDB before giving up
|
||||
* (20 × 250ms ≈ 5s). The stub can surface in the file list a beat before its
|
||||
@@ -81,14 +132,66 @@ export function usePolicyAutoRun(): void {
|
||||
// run so a folder-watch burst opens the modal once, not once per file.
|
||||
const firedLimitModal = useRef<Set<string>>(new Set());
|
||||
|
||||
const onRunFinished = useCallback((view: PolicyRunView) => {
|
||||
const code = view.errorCode;
|
||||
if (code !== "PAYG_LIMIT_REACHED" && code !== "FEATURE_DEGRADED") return;
|
||||
if (firedLimitModal.current.has(view.runId)) return;
|
||||
firedLimitModal.current.add(view.runId);
|
||||
dispatchPaygLimitReached(view.errorSubscribed ?? null);
|
||||
// Latest policies, read from inside the stable retry callback (which has no deps).
|
||||
const policiesRef = useRef(policies);
|
||||
policiesRef.current = policies;
|
||||
// Per-file (dispatchKey) count of consecutive queue-rejection retries, so backoff escalates and
|
||||
// eventually gives up. Survives the run-id changing on each retry; reset on any real outcome.
|
||||
const queueRetries = useRef<Map<string, number>>(new Map());
|
||||
|
||||
// A queue-rejected run is just backpressure — drop the rejected record and fire a fresh run in
|
||||
// its place after a growing backoff (one feed row, not a new one per attempt). Once the budget is
|
||||
// spent, leave the last failure standing so the activity feed offers a manual Retry.
|
||||
const scheduleQueueRetry = useCallback((runId: string) => {
|
||||
const rec = getRun(runId);
|
||||
if (!rec) return;
|
||||
const key = dispatchKey(rec.categoryId, rec.fileId);
|
||||
const attempts = queueRetries.current.get(key) ?? 0;
|
||||
const backendId = policiesRef.current[rec.categoryId]?.backendId;
|
||||
if (attempts >= MAX_QUEUE_RETRIES || !backendId) {
|
||||
queueRetries.current.delete(key);
|
||||
return;
|
||||
}
|
||||
queueRetries.current.set(key, attempts + 1);
|
||||
// Soft-label the row as busy through the backoff window (it's still FAILED underneath).
|
||||
updateRun(runId, { retrying: true });
|
||||
setTimeout(
|
||||
() => {
|
||||
removeRun(runId);
|
||||
void runPolicyOnFile(
|
||||
rec.categoryId,
|
||||
backendId,
|
||||
rec.fileId as FileId,
|
||||
rec.fileName,
|
||||
);
|
||||
},
|
||||
QUEUE_RETRY_BASE_MS * 2 ** attempts,
|
||||
);
|
||||
}, []);
|
||||
|
||||
const onRunFinished = useCallback(
|
||||
(view: PolicyRunView) => {
|
||||
// Transient admission rejection (queue full): back off and retry instead of failing.
|
||||
if (view.errorCode === POLICY_QUEUE_FULL) {
|
||||
scheduleQueueRetry(view.runId);
|
||||
return;
|
||||
}
|
||||
// Any genuine terminal outcome clears the file's retry budget so a later run starts fresh.
|
||||
const finished = getRun(view.runId);
|
||||
if (finished) {
|
||||
queueRetries.current.delete(
|
||||
dispatchKey(finished.categoryId, finished.fileId),
|
||||
);
|
||||
}
|
||||
const code = view.errorCode;
|
||||
if (code !== "PAYG_LIMIT_REACHED" && code !== "FEATURE_DEGRADED") return;
|
||||
if (firedLimitModal.current.has(view.runId)) return;
|
||||
firedLimitModal.current.add(view.runId);
|
||||
dispatchPaygLimitReached(view.errorSubscribed ?? null);
|
||||
},
|
||||
[scheduleQueueRetry],
|
||||
);
|
||||
|
||||
// Dispatch: for each active policy × each session file not yet run, fire a run.
|
||||
useEffect(() => {
|
||||
if (!POLICIES_ENABLED) return;
|
||||
@@ -328,25 +431,49 @@ export async function runPolicyOnFile(
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll a run's status until it reaches a terminal state (or the cap). Calls {@code onTerminal} once
|
||||
* Poll a run's status until it reaches a terminal state (or the budget). Calls {@code onTerminal} once
|
||||
* with the final view when it terminates — the caller uses that to pop the usage-limit modal when a
|
||||
* run was blocked. Only runs polled this session fire it (terminal runs aren't re-polled), so a
|
||||
* persisted failed run never re-triggers a modal on reload.
|
||||
*/
|
||||
async function poll(
|
||||
export async function poll(
|
||||
runId: string,
|
||||
onTerminal?: (view: PolicyRunView) => void,
|
||||
): Promise<void> {
|
||||
for (let i = 0; i < MAX_POLLS; i++) {
|
||||
let notFoundStreak = 0;
|
||||
// Sized to the server's worst case: each step may run up to STEP_TIMEOUT_MS
|
||||
// before the server itself aborts it, so the budget tracks the real pipeline
|
||||
// length (learned from the first status report) rather than a flat cap that
|
||||
// would quit while a long step is still legitimately running.
|
||||
let budgetMs = DEFAULT_STEP_COUNT * STEP_TIMEOUT_MS + POLL_GRACE_MS;
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < budgetMs) {
|
||||
await delay(POLL_MS);
|
||||
let view;
|
||||
try {
|
||||
view = await getPolicyRun(runId);
|
||||
} catch {
|
||||
continue; // transient — keep trying within the cap.
|
||||
} catch (err) {
|
||||
// The server lost the run's (in-memory) state — a restart, or a poll that
|
||||
// hopped to an instance without it. Tolerate a brief blip, then fail so
|
||||
// the file stops enforcing forever; the user can retry.
|
||||
if (isRunNotFound(err)) {
|
||||
if (++notFoundStreak >= MAX_NOT_FOUND) {
|
||||
failRun(runId, "The enforcement run could no longer be found.");
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
notFoundStreak = 0; // a non-404 error doesn't confirm the run is gone.
|
||||
}
|
||||
continue; // keep trying within the budget.
|
||||
}
|
||||
notFoundStreak = 0;
|
||||
if (view.stepCount > 0) {
|
||||
budgetMs = view.stepCount * STEP_TIMEOUT_MS + POLL_GRACE_MS;
|
||||
}
|
||||
updateRun(runId, {
|
||||
status: view.status,
|
||||
currentStep: view.currentStep,
|
||||
stepCount: view.stepCount,
|
||||
outputs: view.outputs,
|
||||
error: view.error,
|
||||
errorCode: view.errorCode ?? null,
|
||||
@@ -356,4 +483,7 @@ async function poll(
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Budget exhausted without a terminal status — stop here and fail it, so the
|
||||
// file doesn't enforce forever and reloads don't re-poll it.
|
||||
failRun(runId, "Enforcement timed out — the run didn't finish in time.");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user