Line data Source code
1 : import { tryNormalizeNodeExecutionId } from '@comfyorg/graph-model/nodeIdentification'
2 : import { Semaphore, delay, partition, withTimeout } from 'es-toolkit'
3 :
4 : import { t } from '@/i18n'
5 : import { zJobExecutionMetaEntry } from '@comfyorg/api-client/jobTypes'
6 : import type { ExecutionError, JobDetail } from '@comfyorg/api-client/jobTypes'
7 : import { api } from '@/scripts/api'
8 : import { useExecutionStore } from '@/stores/executionStore'
9 : import type { TaskItemImpl } from '@/stores/queueStore'
10 : import { useQueueStore } from '@/stores/queueStore'
11 :
12 : type JobId = JobDetail['id']
13 : type TaskOutput = TaskItemImpl['outputs']
14 : type EndedStatus = Extract<
15 : JobDetail['status'],
16 : 'completed' | 'failed' | 'cancelled'
17 : >
18 :
19 : type JobOutcome =
20 : | {
21 : state: 'ended'
22 : status: EndedStatus
23 : outputs?: TaskOutput
24 : displayNodes?: Map<string, string>
25 : executionError?: ExecutionError
26 : timestamp: number
27 : }
28 : | { state: 'live' }
29 : | { state: 'unknown' }
30 :
31 2355 : const LIVE_STATUSES = new Set<JobDetail['status']>(['pending', 'in_progress'])
32 2355 : const DETAIL_TIMEOUT_MS = 5_000
33 2355 : const DETAIL_RETRY_DELAY_MS = 1_000
34 2355 : const DETAIL_CONCURRENCY = 4
35 2355 : const MAX_UNREADABLE_REFRESHES = 3
36 :
37 0 : function unknownExecutionError(): ExecutionError {
38 0 : return {
39 : node_id: '',
40 : node_type: '',
41 : exception_message: t('errorDialog.executionFailedWhileDisconnected'),
42 : exception_type: '',
43 : traceback: []
44 : }
45 : }
46 :
47 0 : function ownDisplayNode(node: string, metaEntry: unknown) {
48 0 : const entry = zJobExecutionMetaEntry.safeParse(metaEntry)
49 0 : if (!entry.success || !entry.data.display_node) return null
50 0 : const { node_id: metaNode = node, display_node: displayNode } = entry.data
51 0 : return metaNode === node ? tryNormalizeNodeExecutionId(displayNode) : null
52 : }
53 :
54 : /**
55 : * Returns the handler for the `reconnected` api event.
56 : */
57 2355 : export function useReconnectQueueRefresh() {
58 2355 : const queueStore = useQueueStore()
59 2355 : const executionStore = useExecutionStore()
60 2355 : const detailRequests = new Semaphore(DETAIL_CONCURRENCY)
61 2355 : const unreadableRefreshes = new Map<JobId, number>()
62 2355 : let inFlight: Promise<void> | null = null
63 2355 : let reconnects = 0
64 :
65 16 : async function fetchDetail(jobId: JobId): Promise<JobDetail | undefined> {
66 16 : await detailRequests.acquire()
67 16 : const request = new AbortController()
68 16 : try {
69 16 : return await withTimeout(
70 : () => api.getJobDetail(jobId, request.signal),
71 : DETAIL_TIMEOUT_MS
72 : )
73 : } catch {
74 0 : request.abort()
75 0 : return undefined
76 : } finally {
77 16 : detailRequests.release()
78 : }
79 : }
80 :
81 0 : function outcomeTimestamp(
82 : detail: JobDetail | undefined,
83 : historyTask: TaskItemImpl | undefined
84 : ) {
85 0 : const detailTimestamp = detail?.execution_end_time
86 0 : if (detailTimestamp != null) return detailTimestamp
87 0 : return historyTask?.executionEndTimestamp ?? Date.now()
88 : }
89 :
90 0 : function historyPreviewOutputs(historyTask: TaskItemImpl | undefined) {
91 0 : if (!historyTask || historyTask.outputsCount !== 1) return undefined
92 0 : if (historyTask.job.preview_output?.mediaType === 'text') return undefined
93 0 : return historyTask.outputs
94 : }
95 :
96 0 : function displayNodesOf(detail: JobDetail | undefined) {
97 0 : const meta = detail?.execution_meta
98 0 : if (meta == null || typeof meta !== 'object' || Array.isArray(meta)) {
99 0 : return undefined
100 : }
101 0 : const displayNodes = new Map<string, string>()
102 0 : for (const [node, entry] of Object.entries(meta)) {
103 0 : const displayNode = ownDisplayNode(node, entry)
104 0 : if (displayNode) displayNodes.set(node, displayNode)
105 : }
106 0 : return displayNodes
107 : }
108 :
109 0 : function completedOutcome(
110 : detail: JobDetail | undefined,
111 : historyTask: TaskItemImpl | undefined
112 : ): JobOutcome {
113 0 : const outputs = detail?.outputs ?? historyPreviewOutputs(historyTask)
114 0 : if (!detail && !outputs) return { state: 'unknown' }
115 0 : return {
116 : state: 'ended',
117 : status: 'completed',
118 : outputs,
119 : displayNodes: displayNodesOf(detail),
120 : timestamp: outcomeTimestamp(detail, historyTask)
121 : }
122 : }
123 :
124 0 : function nonCompletedOutcome(
125 : status: Exclude<EndedStatus, 'completed'>,
126 : detail: JobDetail | undefined,
127 : historyTask: TaskItemImpl | undefined
128 : ): JobOutcome {
129 0 : return {
130 : state: 'ended',
131 : status,
132 : executionError:
133 : detail?.execution_error ?? historyTask?.executionError ?? undefined,
134 : timestamp: outcomeTimestamp(detail, historyTask)
135 : }
136 : }
137 :
138 16 : async function readOutcome(
139 : jobId: JobId,
140 : historyTask: TaskItemImpl | undefined
141 : ): Promise<JobOutcome> {
142 16 : const detail = await fetchDetail(jobId)
143 16 : const status = detail?.status ?? historyTask?.status
144 16 : if (status !== undefined && LIVE_STATUSES.has(status)) {
145 0 : return { state: 'live' }
146 : }
147 16 : if (status === 'completed') return completedOutcome(detail, historyTask)
148 16 : if (status === 'failed' || status === 'cancelled') {
149 0 : return nonCompletedOutcome(status, detail, historyTask)
150 : }
151 16 : return { state: 'unknown' }
152 : }
153 :
154 8 : async function jobOutcome(
155 : jobId: JobId,
156 : historyTask: TaskItemImpl | undefined
157 : ): Promise<JobOutcome> {
158 8 : const outcome = await readOutcome(jobId, historyTask)
159 8 : if (outcome.state !== 'unknown') return outcome
160 8 : await delay(DETAIL_RETRY_DELAY_MS)
161 8 : return readOutcome(jobId, historyTask)
162 : }
163 :
164 0 : function replayOutputs(
165 : jobId: JobId,
166 : outputs: TaskOutput,
167 : displayNodes: Map<string, string> | undefined
168 : ) {
169 0 : const frames = Object.entries(outputs).map(([node, output]) => ({
170 : node,
171 : output,
172 : displayNode: displayNodes?.get(node) ?? node
173 : }))
174 0 : const [own, expanded] = partition(
175 : frames,
176 : ({ node, displayNode }) => node === displayNode
177 : )
178 0 : for (const { node, output, displayNode } of [...own, ...expanded]) {
179 0 : api.dispatchCustomEvent('executed', {
180 : node,
181 : display_node: displayNode,
182 : output,
183 : prompt_id: jobId
184 : })
185 : }
186 : }
187 :
188 0 : function dispatchSuccess(jobId: JobId, timestamp: number) {
189 0 : api.dispatchCustomEvent('execution_success', {
190 : prompt_id: jobId,
191 : timestamp
192 : })
193 : }
194 :
195 0 : function dispatchInterrupted(
196 : jobId: JobId,
197 : timestamp: number,
198 : error: ExecutionError | undefined
199 : ) {
200 0 : api.dispatchCustomEvent('execution_interrupted', {
201 : prompt_id: jobId,
202 : timestamp,
203 : node_id: error?.node_id ?? '',
204 : node_type: error?.node_type ?? '',
205 : executed: error?.executed ?? []
206 : })
207 : }
208 :
209 0 : function dispatchError(
210 : jobId: JobId,
211 : timestamp: number,
212 : error: ExecutionError | undefined
213 : ) {
214 0 : const normalizedError = error ?? unknownExecutionError()
215 0 : api.dispatchCustomEvent('execution_error', {
216 : prompt_id: jobId,
217 : timestamp,
218 : node_id: normalizedError.node_id,
219 : node_type: normalizedError.node_type,
220 : executed: normalizedError.executed ?? [],
221 : exception_message: normalizedError.exception_message,
222 : exception_type: normalizedError.exception_type,
223 : traceback: normalizedError.traceback,
224 : current_inputs: normalizedError.current_inputs,
225 : current_outputs: normalizedError.current_outputs
226 : })
227 : }
228 :
229 0 : function dispatchTerminal(jobId: JobId, outcome: JobOutcome) {
230 0 : if (outcome.state !== 'ended') return
231 0 : if (outcome.status === 'completed') {
232 0 : dispatchSuccess(jobId, outcome.timestamp)
233 0 : return
234 : }
235 :
236 0 : const error = outcome.executionError
237 0 : if (
238 : outcome.status === 'cancelled' ||
239 : error?.exception_type === 'InterruptProcessingException'
240 : ) {
241 0 : dispatchInterrupted(jobId, outcome.timestamp, error)
242 0 : return
243 : }
244 0 : dispatchError(jobId, outcome.timestamp, error)
245 : }
246 :
247 6 : function isFollowed(jobId: JobId) {
248 6 : return (
249 : jobId in executionStore.queuedJobs || executionStore.activeJobId === jobId
250 : )
251 : }
252 :
253 0 : function replayOutcome(jobId: JobId, outcome: JobOutcome) {
254 0 : if (outcome.state !== 'ended' || !isFollowed(jobId)) return
255 0 : if (outcome.outputs) {
256 0 : replayOutputs(jobId, outcome.outputs, outcome.displayNodes)
257 : }
258 0 : dispatchTerminal(jobId, outcome)
259 : }
260 :
261 8 : function giveUpOnUnreadable(jobId: JobId): boolean {
262 8 : const refreshes = (unreadableRefreshes.get(jobId) ?? 0) + 1
263 8 : if (refreshes < MAX_UNREADABLE_REFRESHES) {
264 6 : unreadableRefreshes.set(jobId, refreshes)
265 6 : return false
266 : }
267 2 : unreadableRefreshes.delete(jobId)
268 2 : return true
269 : }
270 :
271 8 : async function refresh() {
272 8 : const followedIds = new Set(Object.keys(executionStore.queuedJobs))
273 8 : if (executionStore.activeJobId) {
274 5 : followedIds.add(executionStore.activeJobId)
275 : }
276 8 : const followed = [...followedIds]
277 8 : await queueStore.updateFresh()
278 6 : const activeJobIds = new Set([
279 : ...queueStore.runningTasks.map((t) => t.jobId),
280 : ...queueStore.pendingTasks.map((t) => t.jobId)
281 : ])
282 6 : const leftQueue = followed.filter((jobId) => !activeJobIds.has(jobId))
283 6 : const historyByJobId = new Map(
284 : queueStore.historyTasks.map((task) => [task.jobId, task])
285 : )
286 6 : const outcomes = await Promise.all(
287 : leftQueue.map((jobId) => jobOutcome(jobId, historyByJobId.get(jobId)))
288 : )
289 :
290 6 : const unreadable: JobId[] = []
291 6 : leftQueue.forEach((jobId, i) => {
292 8 : const outcome = outcomes[i]
293 8 : if (outcome.state === 'live') {
294 0 : activeJobIds.add(jobId)
295 0 : unreadableRefreshes.delete(jobId)
296 0 : return
297 : }
298 8 : if (outcome.state === 'unknown') {
299 8 : if (giveUpOnUnreadable(jobId)) unreadable.push(jobId)
300 6 : else if (historyByJobId.has(jobId)) activeJobIds.add(jobId)
301 8 : return
302 : }
303 0 : unreadableRefreshes.delete(jobId)
304 0 : replayOutcome(jobId, outcome)
305 : })
306 :
307 6 : const currentJobId = executionStore.activeJobId
308 8 : if (!currentJobId || followed.includes(currentJobId)) {
309 6 : executionStore.clearActiveJobIfStale(activeJobIds)
310 : }
311 6 : executionStore.forgetFinishedJobs(unreadable)
312 6 : for (const jobId of unreadableRefreshes.keys()) {
313 6 : if (!isFollowed(jobId)) unreadableRefreshes.delete(jobId)
314 : }
315 : }
316 :
317 8 : async function refreshUntilSettled() {
318 8 : let handled: number
319 8 : do {
320 8 : handled = reconnects
321 8 : await refresh()
322 : } while (handled !== reconnects)
323 : }
324 :
325 2355 : return async function refreshOnReconnect() {
326 8 : reconnects += 1
327 8 : if (inFlight) return inFlight
328 8 : inFlight = refreshUntilSettled().finally(() => {
329 6 : inFlight = null
330 : })
331 8 : return inFlight
332 : }
333 : }
|