Line data Source code
1 : import { uniqBy } from 'es-toolkit'
2 : import MiniSearch from 'minisearch'
3 : import type { SearchResult } from 'minisearch'
4 :
5 : import { searchRankBoost } from './templateRanking'
6 :
7 : export interface SearchableTemplate {
8 : name: string
9 : title?: string
10 : description?: string
11 : localizedTitle?: string
12 : localizedDescription?: string
13 : tags?: string[]
14 : models?: string[]
15 : searchRank?: number
16 : usage?: number
17 : }
18 :
19 : // MiniSearch serializes the index but not the search options, so the tokenizer
20 : // and field list live here and are used at both index and query time.
21 :
22 2355 : const SEARCH_FIELDS = [
23 : 'title',
24 : 'description',
25 : 'tags',
26 : 'models',
27 : 'name'
28 : ] as const
29 :
30 : // Usage only reorders hits within this fraction of the top score, so popularity
31 : // never overrides a clearly-better text match. 5% tuned empirically.
32 2355 : const USAGE_TIEBREAK_BAND = 0.05
33 :
34 : // ~6 tiebreak bands of headroom: enough to lift a near-equal match, never
35 : // enough to surface a weak one.
36 2355 : const SEARCH_RANK_RELEVANCE_WEIGHT = 0.3
37 :
38 : // Script-matched so spaced neighbors like Korean fall to the word tokenizer.
39 2355 : const CJK = /[\p{scx=Han}\p{scx=Hiragana}\p{scx=Katakana}]/u
40 2355 : const CJK_RUN = new RegExp(`${CJK.source}+`, 'gu')
41 :
42 : // Unigrams + bigrams so any substring of an unspaced run lands on a token.
43 39 : function cjkGrams(word: string): string[] {
44 39 : const grams: string[] = []
45 39 : for (const run of word.match(CJK_RUN) ?? []) {
46 0 : const characters = run.split('')
47 0 : grams.push(...characters)
48 0 : for (let i = 1; i < characters.length; i++) {
49 0 : grams.push(characters[i - 1] + characters[i])
50 : }
51 : }
52 39 : return grams
53 : }
54 :
55 : /**
56 : * Emits sub-parts so a term matches however it's typed: `-`/`_` splits, a
57 : * trailing version (`wan2.7` → `wan`, `2.7`), and CJK character grams.
58 : */
59 30 : export function tokenize(text: string): string[] {
60 30 : const tokens = new Set<string>()
61 30 : for (const word of text.toLowerCase().split(/\s+/).filter(Boolean)) {
62 39 : for (const gram of cjkGrams(word)) tokens.add(gram)
63 : // A pure-CJK run has no whole-word token — its grams already cover it.
64 39 : if (CJK.test(word) && !/[a-z0-9]/.test(word)) continue
65 39 : tokens.add(word)
66 39 : for (const part of word.split(/[-_]/)) {
67 57 : if (part) tokens.add(part)
68 : }
69 39 : const version = word.match(/^([a-z][a-z.]*)(\d+(?:\.\d+)*)$/)
70 39 : if (version) {
71 0 : tokens.add(version[1].replace(/\.$/, ''))
72 0 : tokens.add(version[2])
73 : }
74 : }
75 30 : return [...tokens]
76 : }
77 :
78 : // Exact for ≤3-char and digit-bearing terms; otherwise 20% of length, so a typo
79 : // is forgiven but `upscale` can't fuzzy-match the shorter `scale`.
80 3 : export function termFuzziness(term: string): number | false {
81 3 : return term.length <= 3 || /\d/.test(term) ? false : 0.2
82 : }
83 :
84 6 : function searchOptions(combineWith: 'AND' | 'OR' = 'AND') {
85 : // Description demoted below default so an incidental prose mention never
86 : // outranks a real title/model match.
87 6 : return {
88 : boost: { title: 3, models: 2, tags: 2, description: 0.5 },
89 : prefix: true,
90 : fuzzy: termFuzziness,
91 : combineWith,
92 : tokenize
93 : }
94 : }
95 :
96 : // `{X}2{Y}` shorthand expanded by structure (t2i, txt2img, …) rather than one
97 : // entry per spelling.
98 2355 : const MODALITY_STEMS: Record<string, string> = {
99 : t: 'text',
100 : txt: 'text',
101 : text: 'text',
102 : i: 'image',
103 : img: 'image',
104 : image: 'image',
105 : v: 'video',
106 : vid: 'video',
107 : video: 'video',
108 : a: 'audio',
109 : s: 'audio',
110 : m: 'music'
111 : }
112 :
113 2355 : const SAME_MODALITY_EDIT: Record<string, string> = {
114 : image: 'image edit',
115 : video: 'video edit',
116 : audio: 'audio edit'
117 : }
118 :
119 2355 : const ACRONYMS: Record<string, string> = {
120 : cn: 'controlnet'
121 : }
122 :
123 3 : export function expandAbbreviation(token: string): string | null {
124 3 : const lower = token.trim().toLowerCase()
125 3 : if (ACRONYMS[lower]) return ACRONYMS[lower]
126 :
127 3 : const match = lower.match(/^([a-z]+)2([a-z]+)$/)
128 3 : if (!match) return null
129 0 : const left = MODALITY_STEMS[match[1]]
130 0 : const right = MODALITY_STEMS[match[2]]
131 3 : if (!left || !right) return null
132 0 : if (left === right) return SAME_MODALITY_EDIT[left] ?? left
133 0 : return `${left} ${right}`
134 : }
135 :
136 : /** Expands shorthand tokens (`wan i2v` → `wan image video`); null if none expand. */
137 3 : export function expandQuery(query: string): string | null {
138 3 : const expandedTokens = query
139 : .split(/\s+/)
140 : .filter(Boolean)
141 : .map((token) => {
142 3 : const expansion = expandAbbreviation(token.toLowerCase())
143 3 : return { token: expansion ?? token, expanded: expansion !== null }
144 : })
145 3 : if (!expandedTokens.some(({ expanded }) => expanded)) return null
146 0 : return expandedTokens.map(({ token }) => token).join(' ')
147 : }
148 :
149 3 : export function createTemplateSearchIndex(
150 : templates: SearchableTemplate[]
151 : ): MiniSearch<SearchableTemplate> {
152 3 : const index = new MiniSearch<SearchableTemplate>({
153 : idField: 'name',
154 : fields: [...SEARCH_FIELDS],
155 : // Returned on each hit so ranking can read usage and curation without a
156 : // second lookup.
157 : storeFields: ['usage', 'searchRank'],
158 : // Index the localized strings the card actually shows, so a match explains
159 : // a visible result.
160 : extractField: (template, field) => {
161 72 : if (field === 'title') return template.localizedTitle ?? template.title
162 72 : if (field === 'description') {
163 9 : return template.localizedDescription ?? template.description
164 : }
165 54 : const value = template[field as keyof SearchableTemplate]
166 72 : return Array.isArray(value) ? value.join(' ') : (value as string)
167 : },
168 : tokenize,
169 : searchOptions: searchOptions('AND')
170 : })
171 : // Custom-node templates are named by filename, so two packs can share a name;
172 : // a duplicate ID makes MiniSearch throw and leaves search showing everything.
173 3 : index.addAll(uniqBy(templates, (template) => template.name))
174 3 : return index
175 : }
176 :
177 6 : function searchRankMultiplier(searchRank: number | undefined): number {
178 6 : return 1 + searchRankBoost(searchRank) * SEARCH_RANK_RELEVANCE_WEIGHT
179 : }
180 :
181 : // Rank by curated relevance, with usage breaking ties inside a score band.
182 : // Scores are bucketed so the ordering is a stable total order (a pairwise
183 : // relative-band compare is intransitive). log1p dampens heavy-tailed usage.
184 3 : export function rankByRelevanceThenUsage(hits: SearchResult[]): SearchResult[] {
185 3 : const ranked = hits.map((hit) => ({
186 : hit,
187 : score: hit.score * searchRankMultiplier(Number(hit.searchRank ?? 0))
188 : }))
189 3 : const bandSize =
190 : ranked.reduce((max, entry) => Math.max(max, entry.score), 0) *
191 : USAGE_TIEBREAK_BAND
192 3 : const bucket = (score: number) =>
193 : bandSize > 0 ? Math.round(score / bandSize) : 0
194 3 : return ranked
195 : .sort((a, b) => {
196 3 : if (bucket(a.score) !== bucket(b.score)) return b.score - a.score
197 3 : return (
198 : Math.log1p(Number(b.hit.usage ?? 0)) -
199 : Math.log1p(Number(a.hit.usage ?? 0))
200 : )
201 : })
202 : .map((entry) => entry.hit)
203 : }
204 :
205 : /** Ordered template names for a query: literal matches first, then dedup'd expansion matches. */
206 3 : export function searchTemplates(
207 : index: MiniSearch<SearchableTemplate>,
208 : query: string
209 : ): string[] {
210 3 : const trimmed = query.trim()
211 3 : if (!trimmed) return []
212 :
213 3 : const andThenOr = (q: string): SearchResult[] => {
214 3 : const and = index.search(q, searchOptions('AND'))
215 : const hits = and.length > 0 ? and : index.search(q, searchOptions('OR'))
216 3 : return rankByRelevanceThenUsage(hits)
217 : }
218 :
219 3 : const ordered: string[] = []
220 3 : const seen = new Set<string>()
221 3 : const collect = (hits: SearchResult[]) => {
222 3 : for (const hit of hits) {
223 6 : const id = String(hit.id)
224 6 : if (!seen.has(id)) {
225 6 : seen.add(id)
226 6 : ordered.push(id)
227 : }
228 : }
229 : }
230 :
231 3 : collect(andThenOr(trimmed))
232 3 : const expanded = expandQuery(trimmed)
233 3 : if (expanded) collect(andThenOr(expanded))
234 :
235 3 : return ordered
236 : }
|