Line data Source code
1 : import { useIntervalFn } from '@vueuse/core'
2 : import { defineStore } from 'pinia'
3 : import { computed, reactive, ref, watch } from 'vue'
4 :
5 : import type { TaskResponse, TaskStatus } from '@comfyorg/api-client/taskSchema'
6 : import type {
7 : DownloadFileResult,
8 : TaskId,
9 : TaskResult
10 : } from '@/platform/tasks/services/taskService'
11 : import {
12 : TaskNotFoundError,
13 : parseDownloadFileResult,
14 : taskService
15 : } from '@/platform/tasks/services/taskService'
16 : import type { AssetDownloadWsMessage } from '@comfyorg/api-client/executionTypes'
17 : import { api } from '@/scripts/api'
18 : import { t } from '@/i18n'
19 :
20 : export type AssetDownloadStatus =
21 : | TaskStatus
22 : | 'cancellation_pending'
23 : | 'cancellation_unconfirmed'
24 :
25 2356 : const activeStatuses = new Set<AssetDownloadStatus>(['created', 'running'])
26 2356 : const finishedStatuses = new Set<AssetDownloadStatus>([
27 : 'completed',
28 : 'failed',
29 : 'cancelled',
30 : 'cancellation_unconfirmed'
31 : ])
32 2356 : const recheckableStatuses = new Set<AssetDownloadStatus>([
33 : 'created',
34 : 'running',
35 : 'failed',
36 : 'cancellation_pending'
37 : ])
38 2356 : const reconcilableTaskStatuses = new Set<TaskStatus>([
39 : 'completed',
40 : 'failed',
41 : 'cancelled'
42 : ])
43 2356 : const wireTaskStatuses = new Set<string>([
44 : 'created',
45 : 'running',
46 : 'completed',
47 : 'failed',
48 : 'cancelled'
49 : ])
50 :
51 20 : function isDownloadActive(status: AssetDownloadStatus) {
52 20 : return activeStatuses.has(status)
53 : }
54 :
55 23 : export function isDownloadCancelled(status: AssetDownloadStatus) {
56 23 : return (
57 : status === 'cancellation_pending' ||
58 : status === 'cancellation_unconfirmed' ||
59 : status === 'cancelled'
60 : )
61 : }
62 :
63 24 : function isDownloadFinished(status: AssetDownloadStatus) {
64 24 : return finishedStatuses.has(status)
65 : }
66 :
67 20 : function isDownloadRecheckable(status: AssetDownloadStatus) {
68 20 : return recheckableStatuses.has(status)
69 : }
70 :
71 : export interface AssetDownload {
72 : taskId: TaskId
73 : assetName: string
74 : bytesTotal: number
75 : bytesDownloaded: number
76 : progress: number
77 : status: AssetDownloadStatus
78 : lastUpdate: number
79 : assetId?: string
80 : error?: string
81 : modelType?: string
82 : /** `upload` when the file came from the computer rather than a link. */
83 : source?: 'upload'
84 : acknowledged?: boolean
85 : /**
86 : * Reconciliation attempts made since this download entered
87 : * `cancellation_pending` that did not produce a terminal task status.
88 : */
89 : cancellationReconcileAttempts?: number
90 : }
91 :
92 : interface CompletedDownload {
93 : taskId: TaskId
94 : modelType: string
95 : timestamp: number
96 : }
97 : interface DismissedDownload {
98 : modelType?: string
99 : completionNotified: boolean
100 : }
101 2356 : const STALE_THRESHOLD_MS = 10_000
102 2356 : const POLL_INTERVAL_MS = 10_000
103 : /**
104 : * `cancellation_pending` is not a finished status, so the dialog withholds its
105 : * Close control while a download is in it. Bounding how long we wait for the
106 : * backend to confirm keeps a backend that accepts the DELETE and then never
107 : * reports a terminal status from pinning the toast open until a page reload.
108 : */
109 2356 : const MAX_CANCELLATION_RECONCILE_ATTEMPTS = 6
110 2356 : const MAX_TASK_NOT_FOUND_ATTEMPTS = 2
111 2356 : const MAX_FAILED_RECONCILE_ATTEMPTS = 30
112 2356 : const MAX_DISMISSED_DOWNLOADS = 256
113 :
114 0 : function generateDownloadTrackingPlaceholder(
115 : taskId: TaskId,
116 : modelType: string,
117 : assetName: string,
118 : source: AssetDownload['source']
119 : ): AssetDownload {
120 0 : return {
121 : taskId,
122 : modelType,
123 : source,
124 : assetName,
125 : bytesTotal: 0,
126 : bytesDownloaded: 0,
127 : progress: 0,
128 : status: 'created',
129 : lastUpdate: Date.now()
130 : }
131 : }
132 :
133 14 : function trackedFields(
134 : existing: AssetDownload | undefined
135 : ): Pick<AssetDownload, 'modelType' | 'source'> {
136 14 : return { modelType: existing?.modelType, source: existing?.source }
137 : }
138 :
139 14 : function shouldIgnoreDownloadUpdate(
140 : currentStatus: AssetDownloadStatus | undefined,
141 : nextStatus: TaskStatus
142 : ): boolean {
143 14 : if (currentStatus === 'completed') return true
144 14 : if (currentStatus === 'cancelled') return nextStatus !== 'completed'
145 14 : if (currentStatus === 'cancellation_pending') {
146 1 : return !reconcilableTaskStatuses.has(nextStatus)
147 : }
148 14 : if (currentStatus === 'cancellation_unconfirmed') {
149 0 : return activeStatuses.has(nextStatus)
150 : }
151 13 : return false
152 : }
153 :
154 : /**
155 : * Settle a cancellation the backend will never confirm. `cancelled` is not
156 : * immutable in `shouldIgnoreDownloadUpdate`, so a later authoritative
157 : * `completed` still replaces it; the stale error from any premature `failed`
158 : * message is dropped because it no longer describes this entry.
159 : */
160 1 : function finalizeCancellation(download: AssetDownload) {
161 1 : download.status = 'cancelled'
162 1 : download.error = undefined
163 1 : download.lastUpdate = Date.now()
164 : }
165 :
166 : /**
167 : * Record a reconciliation attempt that left a pending cancellation unresolved,
168 : * exposing an unconfirmed but dismissible state once the attempts are
169 : * exhausted. A later WebSocket completion can still replace it, but polling
170 : * stops once the bounded reconciliation window is exhausted.
171 : */
172 1 : function noteAuthoritativePendingCancellation(download: AssetDownload) {
173 1 : if (download.status !== 'cancellation_pending') return
174 :
175 1 : const attempts = (download.cancellationReconcileAttempts ?? 0) + 1
176 1 : download.cancellationReconcileAttempts = attempts
177 1 : if (attempts >= MAX_CANCELLATION_RECONCILE_ATTEMPTS) {
178 0 : download.status = 'cancellation_unconfirmed'
179 0 : download.lastUpdate = Date.now()
180 : }
181 : }
182 :
183 0 : function resultValue<T>(value: T | undefined, fallback: T | undefined) {
184 0 : return value === undefined ? fallback : value
185 : }
186 :
187 0 : function requiredResultValue<T>(value: T | undefined, fallback: T): T {
188 0 : return value === undefined ? fallback : value
189 : }
190 :
191 0 : function createReconciledDownloadMessage(
192 : download: AssetDownload,
193 : task: TaskResponse
194 : ): AssetDownloadWsMessage | undefined {
195 : const result = parseDownloadFileResult(task.result)
196 : // The task status is authoritative even when an older backend returns a
197 : // result shape we cannot parse. Preserve the fields learned from the socket
198 : // so the row can still settle instead of polling forever.
199 0 : const reconciledResult: Partial<DownloadFileResult> = result ?? {}
200 0 : const assetId = resultValue(reconciledResult.asset_id, download.assetId)
201 0 : const assetName = requiredResultValue(
202 : reconciledResult.filename,
203 : download.assetName
204 : )
205 0 : const bytesDownloaded = requiredResultValue(
206 : reconciledResult.bytes_downloaded,
207 : download.bytesDownloaded
208 : )
209 0 : const error = resultValue(task.error_message, reconciledResult.error)
210 0 : return {
211 : task_id: download.taskId,
212 : asset_id: assetId,
213 : asset_name: assetName,
214 : bytes_total: download.bytesTotal,
215 : bytes_downloaded: bytesDownloaded,
216 : progress: task.status === 'completed' ? 100 : download.progress,
217 : status: task.status,
218 : error
219 : }
220 : }
221 :
222 3 : function canCancelDownload(
223 : download: AssetDownload | undefined,
224 : isCancelling: boolean
225 : ) {
226 3 : if (!download || isCancelling) return false
227 3 : return activeStatuses.has(download.status) || download.status === 'failed'
228 : }
229 :
230 3 : function isSettledDownload(download: AssetDownload) {
231 3 : return (
232 : download.status === 'completed' ||
233 : download.status === 'cancelled' ||
234 : download.status === 'cancellation_unconfirmed'
235 : )
236 : }
237 :
238 3 : function beginPendingCancellation(download: AssetDownload) {
239 3 : download.status = 'cancellation_pending'
240 3 : download.error = undefined
241 3 : download.cancellationReconcileAttempts = 0
242 3 : download.lastUpdate = Date.now()
243 : }
244 :
245 0 : function settleUnknownDownloadStatus(
246 : existing: AssetDownload | undefined,
247 : data: AssetDownloadWsMessage
248 : ) {
249 0 : if (
250 : !existing ||
251 : isSettledDownload(existing) ||
252 : existing.status === 'cancellation_pending' ||
253 : existing.status === 'failed'
254 : )
255 0 : return
256 0 : existing.status = 'failed'
257 0 : existing.error = data.error || `Unknown task status: ${data.status}`
258 0 : existing.lastUpdate = Date.now()
259 : }
260 :
261 2356 : export const useAssetDownloadStore = defineStore('assetDownload', () => {
262 2355 : const downloads = ref<Map<string, AssetDownload>>(new Map())
263 2355 : const cancellingTaskIds = ref(new Set<TaskId>())
264 2355 : const reconcilingTasks = new Map<TaskId, Promise<void>>()
265 2355 : const dismissedPendingDownloads = new Map<TaskId, DismissedDownload>()
266 2355 : const taskNotFoundAttempts = new Map<TaskId, number>()
267 2355 : const unavailableTaskIds = reactive(new Set<TaskId>())
268 2355 : const failedReconcileAttempts = reactive(new Map<TaskId, number>())
269 2355 : const settledFailureTaskIds = reactive(new Set<TaskId>())
270 2355 : const lastCompletedDownload = ref<CompletedDownload | null>(null)
271 :
272 2355 : const downloadList = computed(() => Array.from(downloads.value.values()))
273 2355 : const activeDownloads = computed(() =>
274 : downloadList.value.filter((download) => isDownloadActive(download.status))
275 : )
276 2355 : const finishedDownloads = computed(() =>
277 : downloadList.value.filter((download) => isDownloadFinished(download.status))
278 : )
279 2355 : const unacknowledgedDownloads = computed(() =>
280 : finishedDownloads.value.filter(
281 : (d) => d.status === 'completed' && d.assetId && !d.acknowledged
282 : )
283 : )
284 2355 : const sessionDownloadCount = computed(
285 : () => unacknowledgedDownloads.value.length
286 : )
287 2355 : const hasActiveDownloads = computed(() => activeDownloads.value.length > 0)
288 2355 : const hasDownloads = computed(() => downloads.value.size > 0)
289 2355 : const hasPendingCancellation = computed(() =>
290 : downloadList.value.some(
291 : (download) => download.status === 'cancellation_pending'
292 : )
293 : )
294 2355 : const recheckableDownloads = computed(() =>
295 : downloadList.value.filter(
296 : (download) =>
297 : isDownloadRecheckable(download.status) &&
298 : !unavailableTaskIds.has(download.taskId) &&
299 : !settledFailureTaskIds.has(download.taskId)
300 : )
301 : )
302 2355 : const hasRecheckableDownloads = computed(
303 : () => recheckableDownloads.value.length > 0
304 : )
305 :
306 0 : function isDownloadedThisSession(assetId: string): boolean {
307 0 : return unacknowledgedDownloads.value.some((d) => d.assetId === assetId)
308 : }
309 :
310 0 : function acknowledgeAsset(assetId: string) {
311 0 : for (const download of downloads.value.values()) {
312 0 : if (download.assetId === assetId) {
313 0 : download.acknowledged = true
314 : }
315 : }
316 : }
317 :
318 0 : function trackDownload(
319 : taskId: TaskId,
320 : modelType: string,
321 : assetName: string,
322 : { source }: Pick<AssetDownload, 'source'> = {}
323 : ) {
324 0 : const existing = downloads.value.get(taskId)
325 0 : if (existing) {
326 0 : if (existing.modelType) return
327 0 : existing.modelType = modelType
328 0 : existing.source = source
329 0 : if (existing.status === 'completed') {
330 0 : lastCompletedDownload.value = {
331 : taskId,
332 : modelType,
333 : timestamp: Date.now()
334 : }
335 : }
336 0 : return
337 : }
338 :
339 0 : dismissedPendingDownloads.delete(taskId)
340 0 : taskNotFoundAttempts.delete(taskId)
341 0 : unavailableTaskIds.delete(taskId)
342 0 : failedReconcileAttempts.delete(taskId)
343 0 : settledFailureTaskIds.delete(taskId)
344 0 : downloads.value.set(
345 : taskId,
346 : generateDownloadTrackingPlaceholder(taskId, modelType, assetName, source)
347 : )
348 : }
349 :
350 4 : function rememberDismissedDownload(download: AssetDownload) {
351 4 : dismissedPendingDownloads.delete(download.taskId)
352 4 : dismissedPendingDownloads.set(download.taskId, {
353 : modelType: download.modelType,
354 : completionNotified: download.status === 'completed'
355 : })
356 4 : while (dismissedPendingDownloads.size > MAX_DISMISSED_DOWNLOADS) {
357 0 : const oldestTaskId = dismissedPendingDownloads.keys().next().value
358 0 : if (oldestTaskId === undefined) break
359 0 : dismissedPendingDownloads.delete(oldestTaskId)
360 : }
361 : }
362 :
363 15 : function consumeDismissedDownload(data: AssetDownloadWsMessage): boolean {
364 15 : if (!dismissedPendingDownloads.has(data.task_id)) return false
365 :
366 1 : const dismissed = dismissedPendingDownloads.get(data.task_id)
367 15 : if (
368 : data.status === 'completed' &&
369 : dismissed?.modelType &&
370 : !dismissed.completionNotified
371 : ) {
372 0 : lastCompletedDownload.value = {
373 : taskId: data.task_id,
374 : modelType: dismissed.modelType,
375 : timestamp: Date.now()
376 : }
377 0 : dismissed.completionNotified = true
378 : }
379 1 : return true
380 : }
381 :
382 15 : function handleAssetDownload(e: CustomEvent<AssetDownloadWsMessage>) {
383 15 : const data = e.detail
384 15 : if (consumeDismissedDownload(data)) return
385 14 : const existing = downloads.value.get(data.task_id)
386 :
387 : // WebSocket payloads are not runtime-validated at the event boundary.
388 : // Unknown statuses must not create immortal, non-dismissible rows.
389 15 : if (!wireTaskStatuses.has(data.status)) {
390 0 : settleUnknownDownloadStatus(existing, data)
391 0 : return
392 : }
393 :
394 15 : if (shouldIgnoreDownloadUpdate(existing?.status, data.status)) return
395 :
396 14 : if (activeStatuses.has(data.status)) {
397 5 : taskNotFoundAttempts.delete(data.task_id)
398 5 : unavailableTaskIds.delete(data.task_id)
399 5 : failedReconcileAttempts.delete(data.task_id)
400 5 : settledFailureTaskIds.delete(data.task_id)
401 : }
402 :
403 14 : const download: AssetDownload = {
404 : taskId: data.task_id,
405 : assetId: data.asset_id,
406 : assetName: data.asset_name,
407 : bytesTotal: data.bytes_total,
408 : bytesDownloaded: data.bytes_downloaded,
409 : progress: data.progress,
410 : status: data.status,
411 : error: data.error,
412 : lastUpdate: Date.now(),
413 : ...trackedFields(existing)
414 : }
415 :
416 14 : downloads.value.set(data.task_id, download)
417 :
418 15 : if (data.status === 'completed' && download.modelType) {
419 0 : lastCompletedDownload.value = {
420 : taskId: data.task_id,
421 : modelType: download.modelType,
422 : timestamp: Date.now()
423 : }
424 : }
425 : }
426 :
427 1 : function handleTaskNotFound(download: AssetDownload) {
428 1 : if (download.status === 'cancellation_pending') {
429 1 : taskNotFoundAttempts.delete(download.taskId)
430 1 : finalizeCancellation(download)
431 1 : return
432 : }
433 :
434 1 : const attempts = (taskNotFoundAttempts.get(download.taskId) ?? 0) + 1
435 1 : taskNotFoundAttempts.set(download.taskId, attempts)
436 1 : if (attempts < MAX_TASK_NOT_FOUND_ATTEMPTS) {
437 0 : download.lastUpdate = Date.now()
438 0 : return
439 : }
440 0 : taskNotFoundAttempts.delete(download.taskId)
441 0 : unavailableTaskIds.add(download.taskId)
442 0 : download.status = 'failed'
443 0 : download.error = t('progressToast.taskUnavailable')
444 0 : download.lastUpdate = Date.now()
445 : }
446 :
447 3 : async function runReconciliation(download: AssetDownload) {
448 3 : const result = await taskService.getTask(download.taskId)
449 3 : if (downloads.value.get(download.taskId) !== download) return
450 :
451 2 : if (!result.ok) {
452 : // A 404 is authoritative: the task row is gone, so a pending
453 : // cancellation has nothing left to wait for. Other lookup failures are
454 : // transient and must not consume the authoritative reconciliation bound.
455 1 : if (result.error instanceof TaskNotFoundError) {
456 1 : handleTaskNotFound(download)
457 : } else {
458 0 : taskNotFoundAttempts.delete(download.taskId)
459 : }
460 1 : return
461 : }
462 :
463 1 : taskNotFoundAttempts.delete(download.taskId)
464 1 : unavailableTaskIds.delete(download.taskId)
465 1 : const task = result.value
466 3 : if (
467 : download.status === 'cancellation_pending' &&
468 : activeStatuses.has(task.status)
469 : ) {
470 1 : noteAuthoritativePendingCancellation(download)
471 1 : return
472 : }
473 :
474 0 : const message = createReconciledDownloadMessage(download, task)
475 0 : if (!message) {
476 0 : download.lastUpdate = Date.now()
477 0 : noteAuthoritativePendingCancellation(download)
478 0 : return
479 : }
480 0 : if (task.status === 'failed') {
481 0 : const attempts = (failedReconcileAttempts.get(download.taskId) ?? 0) + 1
482 0 : failedReconcileAttempts.set(download.taskId, attempts)
483 0 : if (attempts >= MAX_FAILED_RECONCILE_ATTEMPTS) {
484 0 : settledFailureTaskIds.add(download.taskId)
485 : }
486 : } else {
487 0 : failedReconcileAttempts.delete(download.taskId)
488 0 : settledFailureTaskIds.delete(download.taskId)
489 : }
490 0 : handleAssetDownload(
491 : new CustomEvent('asset_download', {
492 : detail: message
493 : })
494 : )
495 : }
496 :
497 3 : async function reconcileDownload(
498 : download: AssetDownload,
499 : rereadAfterInFlight = false
500 : ) {
501 3 : const inFlight = reconcilingTasks.get(download.taskId)
502 3 : if (inFlight) {
503 0 : if (!rereadAfterInFlight) return
504 0 : await inFlight
505 0 : const latest = downloads.value.get(download.taskId)
506 0 : if (!latest || isSettledDownload(latest)) return
507 0 : download = latest
508 : }
509 :
510 3 : const reconciliation = runReconciliation(download)
511 3 : reconcilingTasks.set(download.taskId, reconciliation)
512 3 : try {
513 3 : await reconciliation
514 : } finally {
515 3 : if (reconcilingTasks.get(download.taskId) === reconciliation) {
516 3 : reconcilingTasks.delete(download.taskId)
517 : }
518 : }
519 : }
520 :
521 5 : async function pollStaleDownloads() {
522 5 : const now = Date.now()
523 5 : const staleDownloads = recheckableDownloads.value.filter(
524 : (d) => now - d.lastUpdate >= STALE_THRESHOLD_MS
525 : )
526 :
527 5 : if (staleDownloads.length === 0) return
528 :
529 3 : await Promise.all(
530 : staleDownloads.map((download) => reconcileDownload(download))
531 : )
532 : }
533 :
534 2355 : const { pause, resume } = useIntervalFn(
535 : () => void pollStaleDownloads(),
536 : POLL_INTERVAL_MS,
537 : { immediate: false }
538 : )
539 :
540 2355 : watch(
541 : hasRecheckableDownloads,
542 : (hasRecheckable) => {
543 2369 : if (hasRecheckable) resume()
544 2361 : else pause()
545 : },
546 : { immediate: true }
547 : )
548 :
549 2355 : api.addEventListener('asset_download', handleAssetDownload)
550 :
551 0 : function clearFinishedDownloads() {
552 0 : for (const download of finishedDownloads.value) {
553 0 : rememberDismissedDownload(download)
554 0 : taskNotFoundAttempts.delete(download.taskId)
555 0 : unavailableTaskIds.delete(download.taskId)
556 0 : failedReconcileAttempts.delete(download.taskId)
557 0 : settledFailureTaskIds.delete(download.taskId)
558 0 : downloads.value.delete(download.taskId)
559 : }
560 : }
561 :
562 4 : function clearDismissibleDownloads() {
563 4 : for (const download of downloadList.value) {
564 4 : if (
565 : isDownloadFinished(download.status) ||
566 : download.status === 'cancellation_pending'
567 : ) {
568 4 : rememberDismissedDownload(download)
569 4 : taskNotFoundAttempts.delete(download.taskId)
570 4 : unavailableTaskIds.delete(download.taskId)
571 4 : failedReconcileAttempts.delete(download.taskId)
572 4 : settledFailureTaskIds.delete(download.taskId)
573 4 : downloads.value.delete(download.taskId)
574 : }
575 : }
576 : }
577 :
578 3 : async function cancelDownload(taskId: TaskId): Promise<TaskResult<boolean>> {
579 : const download = downloads.value.get(taskId)
580 3 : if (!canCancelDownload(download, cancellingTaskIds.value.has(taskId))) {
581 0 : return { ok: true, value: false }
582 : }
583 3 : cancellingTaskIds.value.add(taskId)
584 3 : try {
585 3 : const result = await taskService.cancelTask(taskId)
586 3 : if (!result.ok) return result
587 :
588 3 : const current = downloads.value.get(taskId)
589 3 : if (!current) return { ok: true, value: false }
590 3 : if (isSettledDownload(current)) {
591 0 : return { ok: true, value: result.value === 'cancelling' }
592 : }
593 3 : unavailableTaskIds.delete(taskId)
594 3 : failedReconcileAttempts.delete(taskId)
595 3 : settledFailureTaskIds.delete(taskId)
596 :
597 : // A DELETE 404 can mean the cancellation route is unavailable, not that
598 : // the task is gone. Re-read the task before deciding how to settle it.
599 3 : if (result.value === 'missing') {
600 0 : beginPendingCancellation(current)
601 0 : await reconcileDownload(current, true)
602 0 : const latest = downloads.value.get(taskId)
603 0 : if (!latest) return { ok: true, value: false }
604 0 : if (latest.status === 'cancelled') return { ok: true, value: true }
605 0 : return { ok: true, value: false }
606 : }
607 :
608 : // The backend refused. Re-read the task so the row moves to whatever
609 : // status it is really in, instead of silently re-enabling Cancel.
610 3 : if (result.value === 'not-cancellable') {
611 0 : await reconcileDownload(current, true)
612 0 : return { ok: true, value: false }
613 : }
614 :
615 3 : beginPendingCancellation(current)
616 3 : return { ok: true, value: true }
617 : } finally {
618 3 : cancellingTaskIds.value.delete(taskId)
619 : }
620 : }
621 :
622 2355 : return {
623 : activeDownloads,
624 : finishedDownloads,
625 : hasActiveDownloads,
626 : hasDownloads,
627 : hasPendingCancellation,
628 : downloadList,
629 : lastCompletedDownload,
630 : sessionDownloadCount,
631 : trackDownload,
632 : cancellingTaskIds,
633 : cancelDownload,
634 : clearDismissibleDownloads,
635 : clearFinishedDownloads,
636 : isDownloadedThisSession,
637 : acknowledgeAsset
638 : }
639 : })
|