From d363a1e957352aa7e5acc9ff79cf2da8e8e84627 Mon Sep 17 00:00:00 2001 From: James Brunton Date: Fri, 12 Jun 2026 10:34:11 +0100 Subject: [PATCH] Improve search logic (#6637) # Description of Changes Search has got significantly worse since #6581, where I added all the missing tags for tools that should have been there for months. Turns out that the fuzzy matching search logic has always been way too permissive to match words with Levenshtein distances way too far away from the target word, so long searches include way too much stuff. The new tags just exposed that underlying logic issue. This PR makes the Levenshtein logic much stricter, so it is still tolerant to minor typos in tool names, but doesn't match completely inappropriate strings. --- .../saas/payg/filter/PaygWebMvcConfig.java | 8 +- .../editor/src/core/utils/fuzzySearch.test.ts | 46 +++++++++++ frontend/editor/src/core/utils/fuzzySearch.ts | 27 +++--- .../editor/src/core/utils/toolSearch.test.ts | 82 +++++++++++++++++++ 4 files changed, 149 insertions(+), 14 deletions(-) create mode 100644 frontend/editor/src/core/utils/fuzzySearch.test.ts create mode 100644 frontend/editor/src/core/utils/toolSearch.test.ts diff --git a/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygWebMvcConfig.java b/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygWebMvcConfig.java index 6397bfc27..bce659235 100644 --- a/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygWebMvcConfig.java +++ b/app/saas/src/main/java/stirling/software/saas/payg/filter/PaygWebMvcConfig.java @@ -52,14 +52,14 @@ public class PaygWebMvcConfig implements WebMvcConfigurer { /** * The {@code EntitlementGuard} runs BEFORE the charge interceptor. Spring runs interceptors in * ascending order on the way in and skips a later interceptor's {@code preHandle} (and its - * {@code afterCompletion}) entirely once an earlier one returns {@code false} — so a request the - * guard refuses (over its free allowance / spending cap, or with no subscription to bill) + * {@code afterCompletion}) entirely once an earlier one returns {@code false} — so a request + * the guard refuses (over its free allowance / spending cap, or with no subscription to bill) * short-circuits with its 402 before the charge interceptor ever runs. A blocked request * therefore never opens a process, materialises inputs, or writes a charge: a refused operation * must not bill, and running the guard first guarantees that structurally rather than by * compensating after the fact. Stays above the legacy {@code UnifiedCreditInterceptor} (default - * order 0, only registered under the {@code legacy-credits} profile) so a legacy rejection still - * wins. + * order 0, only registered under the {@code legacy-credits} profile) so a legacy rejection + * still wins. */ public static final int ENTITLEMENT_GUARD_ORDER = 900; diff --git a/frontend/editor/src/core/utils/fuzzySearch.test.ts b/frontend/editor/src/core/utils/fuzzySearch.test.ts new file mode 100644 index 000000000..cf21d58fe --- /dev/null +++ b/frontend/editor/src/core/utils/fuzzySearch.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect } from "vitest"; +import { + scoreMatch, + minScoreForQuery, + isFuzzyMatch, +} from "@app/utils/fuzzySearch"; + +describe("scoreMatch", () => { + it("scores substring matches highest", () => { + expect(scoreMatch("rota", "Rotate")).toBeGreaterThanOrEqual(90); + expect(scoreMatch("priv", "private")).toBeGreaterThanOrEqual( + minScoreForQuery("priv"), + ); + }); + + it("tolerates small typos", () => { + expect(isFuzzyMatch("rotaet", "rotate")).toBe(true); + expect(isFuzzyMatch("comprss", "compress")).toBe(true); + // Typo inside one token of a multi-word target + expect(isFuzzyMatch("watermrk", "Add Watermark")).toBe(true); + }); + + it("rejects unrelated words that share half their letters", () => { + // Regression: the old fallback accepted 50% relative Levenshtein + // similarity, so longer queries matched more junk (rotate vs update is + // edit distance 3 of 6) + expect(isFuzzyMatch("rotate", "update")).toBe(false); + expect(isFuzzyMatch("rotate", "create")).toBe(false); + expect(isFuzzyMatch("rotate", "private")).toBe(false); + expect(isFuzzyMatch("rotate", "isolated")).toBe(false); + expect(isFuzzyMatch("rotate", "automate")).toBe(false); + expect(isFuzzyMatch("rotate", "annotate")).toBe(false); + expect(isFuzzyMatch("rotat", "protect")).toBe(false); + expect(isFuzzyMatch("rotat", "redact")).toBe(false); + expect(isFuzzyMatch("rotat", "contrast")).toBe(false); + expect(isFuzzyMatch("rotat", "annotate")).toBe(false); + }); +}); + +describe("minScoreForQuery", () => { + it("does not loosen the threshold for long queries", () => { + expect(minScoreForQuery("rot")).toBe(40); + expect(minScoreForQuery("rotate")).toBe(30); + expect(minScoreForQuery("orientation")).toBe(30); + }); +}); diff --git a/frontend/editor/src/core/utils/fuzzySearch.ts b/frontend/editor/src/core/utils/fuzzySearch.ts index 9ed2a8cbe..49e4a11e5 100644 --- a/frontend/editor/src/core/utils/fuzzySearch.ts +++ b/frontend/editor/src/core/utils/fuzzySearch.ts @@ -63,20 +63,27 @@ export function scoreMatch(queryRaw: string, targetRaw: string): number { } } - const distance = levenshtein( - query, - target.length > 64 ? target.slice(0, 64) : target, - ); - const maxLen = Math.max(query.length, target.length, 1); - const similarity = 1 - distance / maxLen; // 0..1 - return Math.floor(similarity * 60); // scale below substring scores + // Levenshtein fallback is for typos only: allow a small absolute number of + // edits against the whole target or any single token. Relative similarity is + // not enough here; half the letters of an unrelated word can match (e.g. + // "rotate" vs "update" is edit distance 3 of 6). + const maxEdits = query.length <= 4 ? 1 : 2; + let best = 0; + for (const candidate of [target, ...tokens]) { + if (Math.abs(query.length - candidate.length) > maxEdits) continue; + const distance = levenshtein(query, candidate); + if (distance > maxEdits) continue; + const maxLen = Math.max(query.length, candidate.length, 1); + const similarity = 1 - distance / maxLen; // 0..1 + const score = Math.floor(similarity * 60); // scale below substring scores + if (score > best) best = score; + } + return best; } export function minScoreForQuery(query: string): number { const len = normalizeText(query).length; - if (len <= 3) return 40; - if (len <= 6) return 30; - return 25; + return len <= 3 ? 40 : 30; } // Decide if a target matches a query based on a threshold diff --git a/frontend/editor/src/core/utils/toolSearch.test.ts b/frontend/editor/src/core/utils/toolSearch.test.ts new file mode 100644 index 000000000..f30dc3f4d --- /dev/null +++ b/frontend/editor/src/core/utils/toolSearch.test.ts @@ -0,0 +1,82 @@ +import { describe, it, expect } from "vitest"; +import { filterToolRegistryByQuery } from "@app/utils/toolSearch"; +import { + ToolCategoryId, + SubcategoryId, + ToolRegistry, + ToolRegistryEntry, +} from "@app/data/toolsTaxonomy"; + +function makeEntry(name: string, tags: string): ToolRegistryEntry { + return { + icon: null, + name, + component: null, + description: "", + categoryId: ToolCategoryId.STANDARD_TOOLS, + subcategoryId: SubcategoryId.GENERAL, + automationSettings: null, + synonyms: tags.split(","), + }; +} + +// Real names and tags from the en-GB translations; these tools previously +// polluted results for partial queries like "rotat" and "rotate" +const registry: Partial = { + rotate: makeEntry( + "Rotate", + "turn,flip,orient,rotate,orientation,landscape,portrait,90 degrees,180 degrees,clockwise,anticlockwise,counter-clockwise,fix orientation", + ), + addPassword: makeEntry( + "Add Password", + "encrypt,password,lock,secure,protect,security,encryption,safeguard,confidential,private,restrict access", + ), + changeMetadata: makeEntry( + "Change Metadata", + "edit,modify,update,metadata,properties,document properties,author,title,subject,keywords,creator,producer,info,document info,file properties", + ), + scannerEffect: makeEntry( + "Scanner Effect", + "scan,simulate,create,fake scan,look scanned,scanner effect,make look scanned,photocopy effect,simulate scanner,realistic scan", + ), + adjustContrast: makeEntry( + "Adjust Colours/Contrast", + "contrast,brightness,saturation,adjust colors,color correction,enhance,lighten,darken,improve quality,color balance,hue,vibrance", + ), + annotate: makeEntry( + "Annotate", + "annotate,highlight,draw,markup,comment,notes,review,redline,feedback,markup tools,sticky notes,shapes,arrows,text box,freehand", + ), + redact: makeEntry( + "Redact", + "censor,blackout,hide,redact,redaction,black out,block out,remove sensitive,hide text,privacy,confidential,GDPR,PII,sensitive data,permanently remove,cover up,legal redaction", + ), + compare: makeEntry( + "Compare", + "difference,compare,diff,compare PDFs,compare documents,find differences,show differences,changes,what changed,track changes,revisions,version compare,side by side,contrast,delta", + ), +}; + +function idsFor(query: string): string[] { + return filterToolRegistryByQuery(registry, query).map(({ item: [id] }) => id); +} + +describe("filterToolRegistryByQuery", () => { + it("returns everything for an empty query", () => { + expect(idsFor("")).toHaveLength(Object.keys(registry).length); + }); + + it("matches tools by tag substring", () => { + expect(idsFor("protect")).toEqual(["addPassword"]); + expect(idsFor("orientation")).toEqual(["rotate"]); + }); + + it("only returns Rotate for every prefix of 'rotate'", () => { + // Regression: "rotat" used to also pull in Redact, Annotate, Compare and + // Adjust Colours/Contrast, and "rotate" additionally Scanner Effect, + // Change Metadata, Add Password and Air-gapped Setup + expect(idsFor("rota")).toEqual(["rotate"]); + expect(idsFor("rotat")).toEqual(["rotate"]); + expect(idsFor("rotate")).toEqual(["rotate"]); + }); +});