Line data Source code
1 : import { useChainCallback } from '@/composables/functional/useChainCallback'
2 : import { t } from '@/i18n'
3 : import type { LGraphNode } from '@/lib/litegraph/src/litegraph'
4 : import { useCanvasInteractions } from '@/renderer/core/canvas/useCanvasInteractions'
5 : import { useTelemetry } from '@comfyorg/telemetry'
6 : import { describeImageLoadFailure } from '@/platform/telemetry/imageFailureDiagnostics'
7 : import type { ImageLoadFailureMetadata } from '@comfyorg/telemetry/types'
8 : import { useNodeOutputStore } from '@/stores/nodeOutputStore'
9 : import { getHdrPreviewUrl } from '@/utils/hdrFormatUtil'
10 : import { fitDimensionsToNodeWidth } from '@/utils/imageUtil'
11 :
12 2356 : const VIDEO_WIDGET_NAME = 'video-preview'
13 2356 : const VIDEO_DEFAULT_OPTIONS = {
14 : playsInline: true,
15 : controls: true,
16 : loop: true
17 : } as const
18 2356 : const MEDIA_LOAD_TIMEOUT = 8192
19 2356 : const MAX_RETRIES = 1
20 2356 : const DEFAULT_VIDEO_SIZE = 256
21 2356 : const MEDIA_ERR_SRC_NOT_SUPPORTED = 4
22 :
23 : /** Message for a failed video load, from the media error codes of its attempts. */
24 12 : export function videoFailureMessageKey(errorCodes: (number | undefined)[]) {
25 12 : return errorCodes.includes(MEDIA_ERR_SRC_NOT_SUPPORTED)
26 : ? 'g.videoFormatNotSupported'
27 : : 'g.videoFailedToLoad'
28 : }
29 :
30 : type MediaElement = HTMLImageElement | HTMLVideoElement
31 :
32 : interface NodePreviewOptions<T extends MediaElement> {
33 : loadElement: (url: string) => Promise<T | null>
34 : onLoaded?: (elements: T[]) => void
35 : onFailedLoading?: () => void
36 : /** Which canvas surface this is, so the failure can be attributed. */
37 : telemetrySource: Extract<
38 : ImageLoadFailureMetadata['source'],
39 : 'canvas_node_image' | 'canvas_node_video'
40 : >
41 : }
42 :
43 : interface ShowPreviewOptions {
44 : /** If true, blocks new loading operations until the current operation is complete. */
45 : block?: boolean
46 : }
47 :
48 2356 : const createContainer = () => {
49 17 : const container = document.createElement('div')
50 17 : container.classList.add('comfy-img-preview')
51 17 : return container
52 : }
53 :
54 2356 : const createTimeout = (ms: number) =>
55 : new Promise<null>((resolve) => setTimeout(() => resolve(null), ms))
56 :
57 : /** Distinguishes "the load stalled" from "the load was rejected"; both otherwise yield `null`. */
58 2356 : const MEDIA_LOAD_TIMED_OUT = Symbol('media-load-timed-out')
59 :
60 2356 : const useNodePreview = <T extends MediaElement>(
61 : node: LGraphNode,
62 : options: NodePreviewOptions<T>
63 : ) => {
64 334 : const { loadElement, onLoaded, onFailedLoading, telemetrySource } = options
65 334 : const nodeOutputStore = useNodeOutputStore()
66 :
67 334 : const loadElementWithTimeout = async (
68 : url: string,
69 : retryCount = 0
70 : ): Promise<T | null> => {
71 : // A stalled load and a rejected one both arrive here as `null`, so the
72 : // timeout resolves a sentinel instead: a 401 and an 8s hang are different
73 : // problems and must not collapse into the same event. The sentinel is
74 : // compared rather than a flag set from the timeout's callback — that flag
75 : // would record "the timeout eventually fired", which is true even when the
76 : // load lost the race by milliseconds.
77 272 : const raced = await Promise.race([
78 : loadElement(url),
79 : createTimeout(MEDIA_LOAD_TIMEOUT).then(() => MEDIA_LOAD_TIMED_OUT)
80 : ])
81 260 : const timedOut = raced === MEDIA_LOAD_TIMED_OUT
82 272 : const result: T | null = timedOut ? null : (raced as T | null)
83 :
84 272 : if (result === null && retryCount < MAX_RETRIES) {
85 69 : return loadElementWithTimeout(url, retryCount + 1)
86 : }
87 :
88 205 : if (result === null) {
89 : // Reported once per URL, after retries are spent — a report per attempt
90 : // would inflate the count by exactly `MAX_RETRIES + 1`.
91 63 : void describeImageLoadFailure(url).then((diagnostics) => {
92 60 : useTelemetry()?.trackImageLoadFailed({
93 : ...diagnostics,
94 : source: telemetrySource,
95 : attempts: retryCount + 1,
96 : timed_out: timedOut
97 : })
98 : })
99 : }
100 :
101 191 : return result
102 : }
103 :
104 334 : const loadElements = async (urls: string[]) =>
105 : Promise.all(urls.map((url) => loadElementWithTimeout(url)))
106 :
107 : /**
108 : * Displays media element(s) on the node.
109 : */
110 210 : function showPreview(options: ShowPreviewOptions = {}) {
111 210 : if (node.isLoading) return
112 :
113 210 : const outputUrls = nodeOutputStore.getNodeImageUrls(node)
114 210 : if (!outputUrls?.length) return
115 :
116 208 : if (options.block) node.isLoading = true
117 203 : const { images, preview } = node
118 :
119 203 : loadElements(outputUrls)
120 : .then((elements) => {
121 191 : if (node.images !== images || node.preview !== preview) return
122 121 : const validElements = elements.filter(
123 : (el): el is NonNullable<Awaited<T>> => el !== null
124 : )
125 191 : if (validElements.length) {
126 82 : onLoaded?.(validElements)
127 : } else {
128 39 : onFailedLoading?.()
129 : }
130 191 : node.graph?.setDirtyCanvas(true)
131 : })
132 : .catch(() => {
133 0 : onFailedLoading?.()
134 : })
135 : .finally(() => {
136 191 : node.isLoading = false
137 : })
138 : }
139 :
140 334 : return {
141 : showPreview
142 : }
143 : }
144 :
145 : /**
146 : * Attaches a preview image to a node.
147 : */
148 2356 : export const useNodeImage = (node: LGraphNode, callback?: () => void) => {
149 308 : node.previewMediaType = 'image'
150 :
151 308 : const loadElement = (url: string): Promise<HTMLImageElement | null> =>
152 : new Promise((resolve) => {
153 224 : const img = new Image()
154 224 : img.onload = () => resolve(img)
155 224 : img.onerror = () => resolve(null)
156 224 : const previewUrl = getHdrPreviewUrl(url)
157 224 : if (previewUrl) img.dataset.fileUrl = url
158 224 : img.src = previewUrl ?? url
159 : })
160 :
161 308 : const onLoaded = (elements: HTMLImageElement[]) => {
162 59 : node.imageIndex = null
163 59 : node.imgs = elements
164 59 : callback?.()
165 : }
166 :
167 308 : return useNodePreview(node, {
168 : loadElement,
169 : onLoaded,
170 : telemetrySource: 'canvas_node_image',
171 : onFailedLoading: () => {
172 31 : node.imgs = undefined
173 31 : callback?.()
174 : }
175 : })
176 : }
177 :
178 : /**
179 : * Attaches a preview video to a node.
180 : */
181 2356 : export const useNodeVideo = (node: LGraphNode, callback?: () => void) => {
182 26 : node.previewMediaType = 'video'
183 26 : let minHeight = DEFAULT_VIDEO_SIZE
184 26 : let minWidth = 0
185 26 : let errorCodes: (number | undefined)[] = []
186 :
187 26 : const { handleWheel, handlePointerDown, handlePointerMove } =
188 : useCanvasInteractions()
189 :
190 26 : const findVideoWidget = () =>
191 : node.widgets?.find((w) => w.name === VIDEO_WIDGET_NAME)
192 :
193 26 : const setMinDimensions = (video: HTMLVideoElement) => {
194 23 : const shownWidth = findVideoWidget()?.computeLayoutSize?.(node).minWidth
195 23 : const { minHeight: calculatedHeight, minWidth: calculatedWidth } =
196 : fitDimensionsToNodeWidth(
197 : video.videoWidth,
198 : video.videoHeight,
199 : shownWidth || node.size[0] || DEFAULT_VIDEO_SIZE
200 : )
201 :
202 23 : minWidth = calculatedWidth
203 23 : minHeight = calculatedHeight
204 : }
205 :
206 26 : const loadElement = (url: string): Promise<HTMLVideoElement | null> =>
207 : new Promise((resolve) => {
208 48 : const video = document.createElement('video')
209 48 : Object.assign(video, VIDEO_DEFAULT_OPTIONS)
210 :
211 48 : video.onloadeddata = () => {
212 23 : setMinDimensions(video)
213 23 : resolve(video)
214 : }
215 48 : video.onerror = () => {
216 22 : errorCodes.push(video.error?.code)
217 22 : resolve(null)
218 : }
219 48 : video.src = url
220 : })
221 :
222 26 : const addVideoDomWidget = (container: HTMLElement) => {
223 : const hasWidget = node.widgets?.some((w) => w.name === VIDEO_WIDGET_NAME)
224 17 : if (!hasWidget) {
225 17 : const widget = node.addDOMWidget(VIDEO_WIDGET_NAME, 'video', container, {
226 : surfaces: { canvas: 'shown', vueNode: 'never', panel: 'never' },
227 : hideOnZoom: false
228 : })
229 17 : widget.serialize = false
230 17 : widget.computeLayoutSize = () => ({ minHeight, minWidth })
231 :
232 17 : const controller = new AbortController()
233 17 : const { signal } = controller
234 17 : container.addEventListener('wheel', handleWheel, { signal })
235 17 : container.addEventListener('pointermove', handlePointerMove, { signal })
236 17 : container.addEventListener('pointerdown', handlePointerDown, { signal })
237 :
238 17 : widget.onRemove = useChainCallback(widget.onRemove, () => {
239 3 : controller.abort()
240 : })
241 : }
242 : }
243 :
244 26 : const showInContainer = (element: HTMLElement) => {
245 31 : if (!node.videoContainer) {
246 17 : node.videoContainer = createContainer()
247 17 : addVideoDomWidget(node.videoContainer)
248 : }
249 31 : node.videoContainer.replaceChildren(element)
250 : }
251 :
252 26 : const onLoaded = (videoElements: HTMLVideoElement[]) => {
253 23 : showInContainer(videoElements[0])
254 23 : const widget = findVideoWidget()
255 23 : if (widget) widget.computeLayoutSize = () => ({ minHeight, minWidth })
256 23 : errorCodes = []
257 23 : callback?.()
258 : }
259 :
260 26 : return useNodePreview(node, {
261 : loadElement,
262 : onLoaded,
263 : telemetrySource: 'canvas_node_video',
264 : onFailedLoading: () => {
265 8 : const message = document.createElement('p')
266 8 : message.className = 'p-2 text-center text-sm'
267 8 : message.textContent = t(videoFailureMessageKey(errorCodes))
268 8 : errorCodes = []
269 8 : showInContainer(message)
270 8 : callback?.()
271 : }
272 : })
273 : }
|