Line data Source code
1 : import { until, useEventListener } from '@vueuse/core'
2 : import { delay } from 'es-toolkit'
3 : import { effectScope, ref } from 'vue'
4 :
5 : import { combineAbortSignals } from '@comfyorg/api-client/abortSignal'
6 : import { useFeatureFlags } from '@/composables/useFeatureFlags'
7 : import { useUserBusy } from '@/platform/customNodes/useUserBusy'
8 : import { reportError } from '@comfyorg/telemetry/reportError'
9 : import { useWorkflowService } from '@/platform/workflow/core/services/workflowService'
10 : import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore'
11 : import { couldBootOnDeployment } from '@/platform/workspace/composables/waitForDeploymentPick'
12 : import { useApp } from '@/scripts/appInstance'
13 : import type { StartupHookFailure } from '@/platform/extensions/services/extensionService'
14 : import { useExtensionService } from '@/platform/extensions/services/extensionService'
15 : import { useExtensionStore } from '@/platform/extensions/stores/extensionStore'
16 : import { useWorkspaceStore } from '@/stores/workspaceStore'
17 : import type { ComfyExtension } from '@/types/comfy'
18 : import { isNonNullish } from '@/utils/typeGuardUtil'
19 :
20 : import type {
21 : CpuServerReading,
22 : CpuServerStep,
23 : CpuServerWatch
24 : } from './cpuServerStep'
25 : import { isBeingSetUp, watchCpuServer } from './cpuServerStep'
26 : import {
27 : showFailedToast,
28 : showReadyToast,
29 : showStartingToast
30 : } from './lateToasts'
31 :
32 : interface LateScriptFailure {
33 : ext: string
34 : error: unknown
35 : }
36 :
37 : interface ScriptLoader {
38 : importScript: (
39 : ext: string,
40 : options?: { retry?: boolean }
41 : ) => Promise<LateScriptFailure | undefined>
42 : reportFailures: (failures: LateScriptFailure[]) => void
43 : }
44 :
45 2355 : const LATE_CUSTOM_NODE_WAIT_MS = 2000
46 2355 : const LATE_CUSTOM_NODE_RETRY_MS = 5000
47 2355 : const CPU_SERVER_FIRST_ANSWER_WAIT_MS = 1000
48 2355 : const ARTIST_PAUSE_TIMEOUT_MS = 30_000
49 2355 : const LATE_CUSTOM_NODE_ERROR_TYPE = 'late_extension_failed'
50 :
51 2355 : const MODULE_FETCH_FAILURES = [
52 : 'Failed to fetch dynamically imported module',
53 : 'error loading dynamically imported module'
54 : ]
55 :
56 0 : function messageOf(error: unknown): string {
57 0 : if (error instanceof Error) return error.message
58 0 : return typeof error === 'string' ? error : ''
59 : }
60 :
61 0 : function fetchFailureUrl(error: unknown): string | undefined {
62 0 : const message = messageOf(error)
63 0 : const wording = MODULE_FETCH_FAILURES.find((prefix) =>
64 : message.startsWith(`${prefix}: `)
65 : )
66 0 : return wording === undefined
67 : ? undefined
68 : : message.slice(wording.length + ': '.length)
69 : }
70 :
71 0 : function namesExactly(url: string, error: unknown): boolean {
72 0 : const failedUrl = fetchFailureUrl(error)
73 0 : return (
74 : failedUrl !== undefined &&
75 : (failedUrl === url || failedUrl === new URL(url, document.baseURI).href)
76 : )
77 : }
78 :
79 2355 : class ScriptNotFetched extends Error {
80 : constructor(failure: unknown) {
81 0 : super(messageOf(failure), { cause: failure })
82 0 : this.name = 'ScriptNotFetched'
83 : }
84 : }
85 :
86 0 : function isModuleFetchFailure(error: unknown): boolean {
87 0 : return error instanceof ScriptNotFetched
88 : }
89 :
90 0 : function ownFailure(url: string, error: unknown): unknown {
91 0 : return namesExactly(url, error) ? new ScriptNotFetched(error) : error
92 : }
93 :
94 0 : function swallowedFailure(url: string, swallowed: unknown[]): Error {
95 0 : const named = swallowed.find((error) => namesExactly(url, error))
96 0 : return named === undefined
97 : ? new Error(`Could not import ${url}`)
98 : : new ScriptNotFetched(named)
99 : }
100 :
101 : /** Imports `url` through `load`; a failure counts as not fetched only when it names exactly `url`. */
102 0 : export async function importNoticingFetchFailure(
103 : url: string,
104 : load: (url: string) => Promise<unknown>
105 : ): Promise<void> {
106 0 : const swallowed: unknown[] = []
107 0 : const stopListening = useEventListener(
108 : window,
109 : 'vite:preloadError',
110 : (event) => {
111 0 : const payload: unknown = event.payload
112 0 : swallowed.push(payload)
113 : }
114 : )
115 0 : try {
116 0 : if ((await load(url)) !== undefined) return
117 : } catch (error) {
118 0 : throw ownFailure(url, error)
119 : } finally {
120 0 : stopListening()
121 : }
122 0 : throw swallowedFailure(url, swallowed)
123 : }
124 :
125 : /** Whether startup may stop waiting for slow custom node scripts. */
126 2355 : export function lateLoadingApplies(): boolean {
127 2355 : return (
128 : useFeatureFlags().flags.lateCustomNodesEnabled && couldBootOnDeployment()
129 : )
130 : }
131 :
132 0 : function watchLateWork(signal: AbortSignal, onAbort: () => void) {
133 0 : const scope = effectScope(true)
134 0 : const state = scope.run(() => {
135 0 : const aborted = ref(signal.aborted)
136 0 : const artistBusy = useUserBusy()
137 0 : if (signal.aborted) onAbort()
138 0 : useEventListener(signal, 'abort', () => {
139 0 : aborted.value = true
140 0 : onAbort()
141 : })
142 0 : return { aborted, artistBusy }
143 : })
144 0 : return {
145 : isAborted: () => state?.aborted.value ?? true,
146 : isArtistBusy: () => state?.artistBusy.value ?? false,
147 : stop: () => scope.stop()
148 : }
149 : }
150 :
151 0 : async function rebuildOpenWorkflow() {
152 0 : const workflow = useWorkflowStore().activeWorkflow
153 0 : const baseline = workflow?.changeTracker.initialState
154 0 : const wasModified = workflow?.isModified ?? false
155 0 : await useWorkflowService().reloadCurrentWorkflow()
156 0 : if (!workflow || !baseline) return
157 0 : workflow.changeTracker.initialState = baseline
158 0 : workflow.isModified = wasModified
159 : }
160 :
161 0 : function reportLateCustomNodeFailure(
162 : extension: ComfyExtension,
163 : step: string,
164 : error: unknown
165 : ) {
166 0 : reportError(error, {
167 : errorType: LATE_CUSTOM_NODE_ERROR_TYPE,
168 : surface: 'platform',
169 : level: 'warning',
170 : tags: { extension: extension.name, step }
171 : })
172 : }
173 :
174 0 : function reportStartupHookFailures(failures: StartupHookFailure[]) {
175 0 : for (const { extension, method, error } of failures)
176 0 : reportLateCustomNodeFailure(extension, method, error)
177 : }
178 :
179 0 : function registerHeldCustomNodes(last: boolean) {
180 0 : const service = useExtensionService()
181 0 : const extensionStore = useExtensionStore()
182 0 : const held = last
183 : ? extensionStore.stopHoldingCustomNodes()
184 : : extensionStore.takeHeldCustomNodes()
185 0 : const registered = held.filter((extension) => {
186 0 : try {
187 0 : service.registerExtensionNow(extension)
188 0 : return true
189 : } catch (error) {
190 0 : reportLateCustomNodeFailure(extension, 'register', error)
191 0 : return false
192 : }
193 : })
194 0 : return { registered, allRegistered: registered.length === held.length }
195 : }
196 :
197 0 : function keepHeldCustomNodesForNextEditor() {
198 0 : registerHeldCustomNodes(true)
199 0 : useExtensionStore().finishLateCustomNodes()
200 : }
201 :
202 0 : function reportNodeDefsFailure(error: unknown) {
203 0 : reportError(error, {
204 : errorType: LATE_CUSTOM_NODE_ERROR_TYPE,
205 : surface: 'platform',
206 : level: 'warning',
207 : tags: { step: 'getNodeDefs' }
208 : })
209 : }
210 :
211 0 : async function fetchNodeDefs() {
212 0 : try {
213 0 : const fetched = await useApp().fetchNodeDefs()
214 0 : if (fetched.ok) return fetched.defs
215 0 : reportNodeDefsFailure(
216 : new Error(`GET /object_info answered ${fetched.status}`)
217 : )
218 : } catch (error) {
219 0 : reportNodeDefsFailure(error)
220 : }
221 : return undefined
222 : }
223 :
224 0 : async function startLateCustomNodes(
225 : extensions: ComfyExtension[]
226 : ): Promise<boolean> {
227 0 : const service = useExtensionService()
228 0 : const failures = await service.invokeStartupHook(extensions, 'init')
229 0 : const defs = await fetchNodeDefs()
230 0 : if (defs)
231 0 : failures.push(
232 : ...(await service.invokeStartupHook(
233 : useExtensionStore().extensions,
234 : 'addCustomNodeDefs',
235 : defs
236 : ))
237 : )
238 0 : else useExtensionStore().oweNodeDefHooks()
239 0 : failures.push(
240 : ...(await service.invokeStartupHook(extensions, 'registerCustomNodes'))
241 : )
242 0 : if (defs) await useApp().reloadNodeDefs(defs)
243 0 : failures.push(...(await service.invokeStartupHook(extensions, 'setup')))
244 0 : reportStartupHookFailures(failures)
245 0 : return (
246 : defs !== undefined &&
247 : failures.every(({ extension }) => !extensions.includes(extension))
248 : )
249 : }
250 :
251 : interface BatchOutcome {
252 : turnedOn: boolean
253 : allStarted: boolean
254 : }
255 :
256 0 : async function runLateBatch(
257 : work: ReturnType<typeof watchLateWork>,
258 : last: boolean
259 : ): Promise<BatchOutcome> {
260 0 : await until(() => !useWorkspaceStore().spinner || work.isAborted()).toBe(true)
261 0 : await until(() => !work.isArtistBusy() || work.isAborted()).toBe(true, {
262 : timeout: ARTIST_PAUSE_TIMEOUT_MS
263 : })
264 0 : if (work.isAborted()) return { turnedOn: false, allStarted: false }
265 :
266 0 : const { registered, allRegistered } = registerHeldCustomNodes(last)
267 0 : const allHooksRan =
268 : registered.length === 0 || (await startLateCustomNodes(registered))
269 0 : if (work.isAborted()) return { turnedOn: false, allStarted: false }
270 0 : if (last) useExtensionStore().finishLateCustomNodes()
271 0 : if (last || registered.length > 0) await rebuildOpenWorkflow()
272 0 : return {
273 : turnedOn: registered.length > 0,
274 : allStarted: allRegistered && allHooksRan
275 : }
276 : }
277 :
278 : interface LateRun {
279 : firstTries: Promise<(LateScriptFailure | undefined)[]>
280 : waiting: Set<string>
281 : loader: ScriptLoader
282 : signal: AbortSignal
283 : server: CpuServerWatch
284 : stopWatchingServer: () => void
285 : }
286 :
287 : interface LateOutcome extends BatchOutcome {
288 : failedAgain: LateScriptFailure[]
289 : }
290 :
291 0 : function isSettingUp(step: CpuServerStep | undefined): boolean {
292 0 : return step === 'creating' || step === 'building'
293 : }
294 :
295 0 : async function retryFetchFailures(
296 : failures: LateScriptFailure[],
297 : { loader, waiting, signal, server }: LateRun
298 : ): Promise<LateScriptFailure[]> {
299 : const waited = await delay(LATE_CUSTOM_NODE_RETRY_MS, { signal }).then(
300 : () => true,
301 : () => false
302 : )
303 0 : if (!waited) return []
304 0 : if (isSettingUp(server.step())) await server.ready
305 0 : if (signal.aborted) return []
306 : const retries = await Promise.all(
307 : failures.map(({ ext }) =>
308 : loader
309 : .importScript(ext, { retry: true })
310 : .finally(() => waiting.delete(ext))
311 : )
312 : )
313 0 : return retries.filter(isNonNullish)
314 : }
315 :
316 0 : async function turnOnInBatches(
317 : run: LateRun,
318 : work: ReturnType<typeof watchLateWork>,
319 : onWaitingChange: () => void
320 : ): Promise<LateOutcome> {
321 0 : const failures = (await run.firstTries).filter(isNonNullish)
322 0 : const fetchFailures = failures.filter(({ error }) =>
323 : isModuleFetchFailure(error)
324 : )
325 0 : const otherFailures = failures.filter(
326 : (failure) => !fetchFailures.includes(failure)
327 : )
328 0 : run.loader.reportFailures(otherFailures)
329 0 : const scriptsLoaded = otherFailures.length === 0
330 0 : if (fetchFailures.length === 0) {
331 0 : const batch = await runLateBatch(work, true)
332 0 : return {
333 : turnedOn: batch.turnedOn,
334 : allStarted: batch.allStarted && scriptsLoaded,
335 : failedAgain: []
336 : }
337 : }
338 :
339 0 : fetchFailures.forEach(({ ext }) => run.waiting.add(ext))
340 0 : onWaitingChange()
341 0 : const retries = retryFetchFailures(fetchFailures, run)
342 0 : const first = await runLateBatch(work, false)
343 0 : const failedAgain = await retries
344 0 : const last = await runLateBatch(work, true)
345 0 : return {
346 : turnedOn: first.turnedOn || last.turnedOn,
347 : allStarted: first.allStarted && last.allStarted && scriptsLoaded,
348 : failedAgain
349 : }
350 : }
351 :
352 0 : function showOutcome({ turnedOn, allStarted, failedAgain }: LateOutcome) {
353 0 : if (failedAgain.length > 0) showFailedToast(failedAgain.map(({ ext }) => ext))
354 0 : else if (turnedOn && allStarted) showReadyToast()
355 : }
356 :
357 0 : async function turnOnLateCustomNodes(run: LateRun, startedAt: number) {
358 0 : const starting = showStartingToast(startedAt, run.waiting, run.server.step)
359 0 : const work = watchLateWork(run.signal, () => {
360 0 : starting.dismiss()
361 0 : keepHeldCustomNodesForNextEditor()
362 : })
363 0 : try {
364 0 : const outcome = await turnOnInBatches(run, work, starting.refresh)
365 0 : run.loader.reportFailures(outcome.failedAgain)
366 0 : if (!work.isAborted()) showOutcome(outcome)
367 : } catch (error) {
368 0 : if (!run.signal.aborted)
369 0 : reportError(error, {
370 : errorType: LATE_CUSTOM_NODE_ERROR_TYPE,
371 : surface: 'platform',
372 : level: 'warning'
373 : })
374 : } finally {
375 0 : run.stopWatchingServer()
376 0 : starting.dismiss()
377 0 : work.stop()
378 0 : if (!work.isAborted()) useExtensionStore().finishLateCustomNodes()
379 : }
380 : }
381 :
382 0 : function startLateRun(
383 : run: Pick<LateRun, 'waiting' | 'loader' | 'signal'>,
384 : firstReading: Promise<CpuServerReading>,
385 : startedAt: number,
386 : firstTries: (server: CpuServerWatch) => LateRun['firstTries']
387 : ) {
388 0 : const watching = new AbortController()
389 0 : const server = watchCpuServer(
390 : firstReading,
391 : combineAbortSignals([run.signal, watching.signal])
392 : )
393 0 : useExtensionStore().holdNewCustomNodes()
394 0 : void turnOnLateCustomNodes(
395 : {
396 : ...run,
397 : firstTries: firstTries(server),
398 : server,
399 : stopWatchingServer: () => watching.abort()
400 : },
401 : startedAt
402 : )
403 : }
404 :
405 : /** Loads custom scripts with a two-second wait; later arrivals are turned on until `signal` aborts. */
406 0 : export async function loadCustomNodesWithDeadline(
407 : scripts: string[],
408 : loader: ScriptLoader,
409 : signal: AbortSignal,
410 : firstReading: Promise<CpuServerReading>
411 : ) {
412 0 : const startedAt = Date.now()
413 0 : const waiting = new Set(scripts)
414 0 : const importAll = () =>
415 : Promise.all(
416 : scripts.map((ext) =>
417 : loader.importScript(ext).finally(() => waiting.delete(ext))
418 : )
419 : )
420 0 : const readingInTime = await Promise.race([
421 : firstReading,
422 : delay(CPU_SERVER_FIRST_ANSWER_WAIT_MS, { signal }).then(
423 : () => undefined,
424 : () => undefined
425 : )
426 : ])
427 0 : if (signal.aborted) return
428 0 : if (isBeingSetUp(readingInTime)) {
429 0 : startLateRun(
430 : { waiting, loader, signal },
431 : firstReading,
432 : startedAt,
433 : (server) => server.ready.then(() => (signal.aborted ? [] : importAll()))
434 : )
435 0 : return
436 : }
437 :
438 0 : const firstTries = importAll()
439 : const settledInTime = await Promise.race([
440 : firstTries.then(() => true),
441 : delay(Math.max(0, startedAt + LATE_CUSTOM_NODE_WAIT_MS - Date.now())).then(
442 : () => false
443 : )
444 : ])
445 0 : if (settledInTime) {
446 0 : const failures = (await firstTries).filter(isNonNullish)
447 0 : if (!failures.some(({ error }) => isModuleFetchFailure(error))) {
448 0 : loader.reportFailures(failures)
449 0 : return
450 : }
451 : }
452 0 : startLateRun(
453 : { waiting, loader, signal },
454 : firstReading,
455 : startedAt,
456 : () => firstTries
457 : )
458 : }
|