Line data Source code
1 : import { nextTick } from 'vue'
2 :
3 : import Load3DViewerContent from '@/extensions/core/load3d/components/Load3dViewerContent.vue'
4 : import { LOAD3D_VIEWER_DIALOG_PROPS } from '@/extensions/core/load3d/components/load3dViewerDialog'
5 : import {
6 : getLoad3dOutputCache,
7 : getLoad3dSceneRevision,
8 : isLoad3dSceneDirty,
9 : markLoad3dSceneDirty,
10 : nodeToLoad3dMap,
11 : setLoad3dOutputCache,
12 : useLoad3d
13 : } from '@/extensions/core/load3d/composables/useLoad3d'
14 : import type { Load3dCachedOutput } from '@/extensions/core/load3d/composables/useLoad3d'
15 : import { createNodeExportMenuItems } from '@/extensions/core/load3d/exportMenuHelper'
16 : import type {
17 : CameraConfig,
18 : CameraState,
19 : LoadFolder,
20 : Model3DInfo
21 : } from '@comfyorg/viewport3d/load3d/interfaces'
22 : import type Load3d from '@comfyorg/viewport3d/load3d/Load3d'
23 : import Load3DConfiguration from '@/extensions/core/load3d/Load3DConfiguration'
24 : import { findNumericWidget } from '@/extensions/core/load3d/findNumericWidget'
25 : import { snapshotLoad3dState } from '@/extensions/core/load3d/load3dSerialize'
26 : import type { Model3DOutput } from '@/extensions/core/load3d/model3dOutput'
27 : import { readModel3DOutput } from '@/extensions/core/load3d/model3dOutput'
28 : import Load3dUtils from '@/extensions/core/load3d/Load3dUtils'
29 : import { t } from '@/i18n'
30 : import type { LGraphNode } from '@/lib/litegraph/src/LGraphNode'
31 : import type { IContextMenuValue } from '@/lib/litegraph/src/types/contextMenu'
32 : import { useToast } from '@/components/ui/toast/toastStore'
33 : import type {
34 : NodeExecutionOutput,
35 : NodeOutputWith
36 : } from '@comfyorg/api-client/executionTypes'
37 : import { reportError } from '@comfyorg/telemetry/reportError'
38 : import type { NodeLocatorId } from '@comfyorg/graph-model/nodeIdentification'
39 : import { getNodeByLocatorId } from '@/utils/graphTraversalUtil'
40 :
41 : type Matrix = number[][]
42 :
43 : /** Extra captures allowed when the scene changes mid-capture. */
44 24 : const MAX_STALE_CAPTURE_RETRIES = 2
45 :
46 : type Load3dPreviewOutput = NodeOutputWith<{
47 : result?: [string?, CameraState?, string?, Matrix?, Matrix?]
48 : }>
49 : import { ComfyApp, app } from '@/scripts/app'
50 : import { useLoad3dService } from '@/extensions/core/load3d/services/load3dService'
51 : import { useDialogStore } from '@/stores/dialogStore'
52 : import type { ComfyExtension } from '@/types/comfy'
53 : import { isLoad3dNode } from '@/utils/litegraphUtil'
54 : import { selectedCanvasNodes } from '@/renderer/core/canvas/litegraph/selectionAdapter'
55 :
56 24 : export const load3dExtension: ComfyExtension = {
57 : name: 'Comfy.Load3D',
58 : commands: [
59 : {
60 : id: 'Comfy.3DViewer.Open3DViewer',
61 : icon: 'icon-[lucide--pencil]',
62 : label: 'Open 3D Viewer (Beta) for Selected Node',
63 : function: () => {
64 0 : const selectedNodes = selectedCanvasNodes(app.canvas)
65 0 : if (selectedNodes.length !== 1) return
66 :
67 0 : const [selectedNode] = selectedNodes
68 0 : if (!isLoad3dNode(selectedNode)) return
69 :
70 0 : ComfyApp.copyToClipspace(selectedNode)
71 0 : ComfyApp.clipspace_return_node = selectedNode
72 :
73 0 : const props = { node: selectedNode }
74 :
75 0 : useDialogStore().showDialog({
76 : key: 'global-load3d-viewer',
77 : title: t('load3d.viewer.title'),
78 : component: Load3DViewerContent,
79 : props: props,
80 : dialogComponentProps: {
81 : ...LOAD3D_VIEWER_DIALOG_PROPS,
82 : onClose: async () => {
83 0 : await useLoad3dService().handleViewerClose(props.node)
84 : }
85 : }
86 : })
87 : }
88 : }
89 : ],
90 : getNodeMenuItems(node: LGraphNode): (IContextMenuValue | null)[] {
91 12 : if (node.constructor.comfyClass !== 'Load3D') return []
92 :
93 12 : return createNodeExportMenuItems(node)
94 : },
95 :
96 : async nodeCreated(node: LGraphNode) {
97 26 : if (node.constructor.comfyClass !== 'Load3D') return
98 :
99 22 : const [oldWidth, oldHeight] = node.size
100 :
101 22 : node.setSize([Math.max(oldWidth, 300), Math.max(oldHeight, 600)])
102 :
103 22 : await nextTick()
104 :
105 22 : useLoad3d(node).onLoad3dReady((load3d) => {
106 21 : const modelWidget = node.widgets?.find((w) => w.name === 'model_file')
107 21 : const width = findNumericWidget(node.widgets, 'width')
108 21 : const height = findNumericWidget(node.widgets, 'height')
109 21 : if (!modelWidget || !width || !height) return
110 :
111 : const cameraConfig = node.properties['Camera Config'] as
112 : | CameraConfig
113 : | undefined
114 21 : const cameraState = cameraConfig?.state
115 :
116 : const config = new Load3DConfiguration(load3d, node.properties)
117 21 : config.configure({
118 : loadFolder: 'input',
119 : modelWidget,
120 : cameraState,
121 : width,
122 : height,
123 : onSceneInvalidated: () => markLoad3dSceneDirty(node)
124 : })
125 : })
126 :
127 22 : useLoad3d(node).waitForLoad3d(() => {
128 21 : const modelWidget = node.widgets?.find((w) => w.name === 'model_file')
129 21 : const width = node.widgets?.find((w) => w.name === 'width')
130 21 : const height = node.widgets?.find((w) => w.name === 'height')
131 21 : const sceneWidget = node.widgets?.find((w) => w.name === 'image')
132 :
133 21 : if (modelWidget && width && height && sceneWidget) {
134 21 : sceneWidget.serializeValue = async () => {
135 3 : const currentLoad3d = nodeToLoad3dMap.get(node)
136 3 : if (!currentLoad3d) {
137 0 : console.error('No load3d instance found for node')
138 0 : return null
139 : }
140 :
141 3 : for (let attempt = 0; ; attempt++) {
142 3 : if (!isLoad3dSceneDirty(node)) {
143 0 : const cached = getLoad3dOutputCache(node)
144 0 : if (cached) return cached
145 : }
146 :
147 3 : const sceneRevision = getLoad3dSceneRevision(node)
148 :
149 : // A model swap (user or agent) may still be loading. Capture the
150 : // scene the queue will actually run, not the one being replaced.
151 3 : await currentLoad3d.whenLoadIdle()
152 :
153 3 : const { camera_info, model_3d_info } = snapshotLoad3dState(
154 : node,
155 : currentLoad3d
156 : )
157 :
158 3 : const {
159 : scene: imageData,
160 : mask: maskData,
161 : normal: normalData
162 : } = await currentLoad3d.captureScene(
163 : width.value as number,
164 : height.value as number
165 : )
166 :
167 3 : const [data, dataMask, dataNormal] = await Promise.all([
168 : Load3dUtils.uploadTempImage(imageData, 'scene'),
169 : Load3dUtils.uploadTempImage(maskData, 'scene_mask'),
170 : Load3dUtils.uploadTempImage(normalData, 'scene_normal')
171 : ])
172 :
173 3 : currentLoad3d.handleResize()
174 :
175 3 : const returnVal: Load3dCachedOutput = {
176 : image: `threed/${data.name} [temp]`,
177 : mask: `threed/${dataMask.name} [temp]`,
178 : normal: `threed/${dataNormal.name} [temp]`,
179 : camera_info,
180 : recording: '',
181 : model_3d_info
182 : }
183 :
184 3 : const recordingData = currentLoad3d.getRecordingData()
185 :
186 3 : if (recordingData) {
187 : const recording = await Load3dUtils.uploadTempImage(
188 : recordingData,
189 : 'recording',
190 : 'mp4'
191 : )
192 0 : returnVal.recording = `threed/${recording.name} [temp]`
193 : }
194 :
195 3 : if (setLoad3dOutputCache(node, returnVal, sceneRevision)) {
196 3 : return returnVal
197 : }
198 :
199 0 : if (attempt >= MAX_STALE_CAPTURE_RETRIES) {
200 0 : reportError(
201 : new Error('Load3D scene did not stabilize during capture'),
202 : {
203 : surface: 'assets',
204 : errorType: 'error_capturing_load3d_scene_unstable'
205 : }
206 : )
207 0 : return null
208 : }
209 : }
210 : }
211 : }
212 : })
213 : }
214 : }
215 :
216 0 : function applyPreview3DOutput(
217 : node: LGraphNode,
218 : result: NonNullable<Load3dPreviewOutput['result']>
219 : ): void {
220 0 : const filePath = result[0]
221 0 : const cameraState = result[1]
222 0 : const bgImagePath = result[2]
223 0 : const extrinsics = result[3]
224 0 : const intrinsics = result[4]
225 0 : if (!filePath) return
226 :
227 0 : const modelWidget = node.widgets?.find((w) => w.name === 'model_file')
228 0 : if (!modelWidget) return
229 :
230 0 : const normalizedPath = filePath.replaceAll('\\', '/')
231 :
232 : // Always re-apply, even when the file path matches: the same model file
233 : // can arrive with a new camera state, background image, or matrices, and
234 : // a path-only guard would silently drop those updates and diverge from
235 : // the active `node.onExecuted` path which always reapplies.
236 0 : modelWidget.value = normalizedPath
237 0 : node.properties['Last Time Model File'] = normalizedPath
238 :
239 0 : useLoad3d(node).waitForLoad3d((load3d) => {
240 : const config = new Load3DConfiguration(load3d, node.properties)
241 0 : config.configure({
242 : loadFolder: 'output',
243 : modelWidget,
244 : cameraState,
245 : bgImagePath,
246 : silentOnNotFound: true
247 : })
248 :
249 0 : if (bgImagePath) void load3d.setBackgroundImage(bgImagePath)
250 :
251 0 : if (extrinsics && intrinsics) {
252 0 : const targetGeneration = load3d.currentLoadGeneration
253 0 : void load3d
254 : .whenLoadIdle()
255 : .then(() => {
256 0 : if (load3d.currentLoadGeneration !== targetGeneration) return
257 0 : load3d.setCameraFromMatrices(extrinsics, intrinsics)
258 : })
259 : .catch((error) => {
260 0 : console.error(
261 : 'Failed to apply camera matrices from Preview3D output:',
262 : error
263 : )
264 : })
265 : }
266 : })
267 : }
268 :
269 24 : export const preview3DExtension: ComfyExtension = {
270 : name: 'Comfy.Preview3D',
271 :
272 : onNodeOutputsUpdated(
273 : nodeOutputs: Record<NodeLocatorId, NodeExecutionOutput>
274 : ) {
275 0 : for (const [locatorId, output] of Object.entries(nodeOutputs)) {
276 0 : const result = (output as Load3dPreviewOutput).result
277 0 : if (!result?.[0]) continue
278 :
279 0 : const node = getNodeByLocatorId(app.rootGraph, locatorId)
280 0 : if (!node || node.constructor.comfyClass !== 'Preview3D') continue
281 :
282 0 : applyPreview3DOutput(node, result)
283 : }
284 : },
285 :
286 : getNodeMenuItems(node: LGraphNode): (IContextMenuValue | null)[] {
287 12 : if (node.constructor.comfyClass !== 'Preview3D') return []
288 :
289 0 : return createNodeExportMenuItems(node)
290 : },
291 :
292 : async nodeCreated(node: LGraphNode) {
293 26 : if (node.constructor.comfyClass !== 'Preview3D') return
294 :
295 4 : const [oldWidth, oldHeight] = node.size
296 :
297 4 : node.setSize([Math.max(oldWidth, 400), Math.max(oldHeight, 550)])
298 :
299 4 : await nextTick()
300 :
301 4 : const onExecuted = node.onExecuted
302 :
303 4 : useLoad3d(node).onLoad3dReady((load3d) => {
304 4 : const modelWidget = node.widgets?.find((w) => w.name === 'model_file')
305 4 : if (!modelWidget) return
306 :
307 4 : const lastTimeModelFile = node.properties['Last Time Model File']
308 4 : if (!lastTimeModelFile) return
309 :
310 3 : modelWidget.value = lastTimeModelFile
311 :
312 : const cameraConfig = node.properties['Camera Config'] as
313 : | CameraConfig
314 : | undefined
315 4 : const cameraState = cameraConfig?.state
316 :
317 : const config = new Load3DConfiguration(load3d, node.properties)
318 4 : config.configure({
319 : loadFolder: 'output',
320 : modelWidget,
321 : cameraState,
322 : silentOnNotFound: true
323 : })
324 : })
325 :
326 4 : useLoad3d(node).waitForLoad3d((load3d) => {
327 4 : const config = new Load3DConfiguration(load3d, node.properties)
328 :
329 4 : const modelWidget = node.widgets?.find((w) => w.name === 'model_file')
330 :
331 4 : if (modelWidget) {
332 4 : node.onExecuted = function (output: Load3dPreviewOutput) {
333 1 : onExecuted?.call(this, output)
334 :
335 1 : const result = output.result
336 1 : const filePath = result?.[0]
337 :
338 1 : if (!filePath) {
339 0 : const msg = t('toastMessages.unableToGetModelFilePath')
340 0 : console.error(msg)
341 0 : useToast().warning(msg)
342 : }
343 :
344 1 : const cameraState = result?.[1]
345 1 : const bgImagePath = result?.[2]
346 1 : const extrinsics = result?.[3]
347 1 : const intrinsics = result?.[4]
348 :
349 1 : const modelFilePath = filePath?.replaceAll('\\', '/')
350 1 : modelWidget.value = modelFilePath
351 :
352 1 : node.properties['Last Time Model File'] = modelFilePath
353 :
354 1 : const settings = {
355 : loadFolder: 'output',
356 : modelWidget: modelWidget,
357 : cameraState: cameraState,
358 : bgImagePath: bgImagePath,
359 : silentOnNotFound: true
360 : }
361 :
362 1 : config.configure(settings)
363 :
364 1 : if (bgImagePath) void load3d.setBackgroundImage(bgImagePath)
365 :
366 1 : if (filePath && extrinsics && intrinsics) {
367 : // configure(settings) above triggered loadModel for this
368 : // execution; capture its generation so that if a newer
369 : // execution queues another load before whenLoadIdle resolves,
370 : // we don't apply this execution's matrices on top of that
371 : // newer model.
372 0 : const targetGeneration = load3d.currentLoadGeneration
373 0 : void load3d
374 : .whenLoadIdle()
375 : .then(() => {
376 0 : if (load3d.currentLoadGeneration !== targetGeneration) return
377 0 : load3d.setCameraFromMatrices(extrinsics, intrinsics)
378 : })
379 : .catch((error) => {
380 0 : console.error(
381 : 'Failed to apply camera matrices from Preview3D output:',
382 : error
383 : )
384 : })
385 : }
386 : }
387 : }
388 : })
389 : }
390 : }
391 :
392 0 : function applyPreview3DAdvancedResult(
393 : node: LGraphNode,
394 : load3d: Load3d,
395 : reported: Model3DOutput,
396 : loadFolder: LoadFolder,
397 : comfyClass: string
398 : ): void {
399 0 : const normalizedPath = reported.filePath.replaceAll('\\', '/')
400 0 : const folder = reported.folder ?? loadFolder
401 0 : node.properties['Last Time Model File'] = normalizedPath
402 0 : node.properties['Last Time Model Folder'] = folder
403 :
404 : const config = new Load3DConfiguration(load3d, node.properties)
405 0 : config.configureForSaveMesh(folder, normalizedPath, {
406 : silentOnNotFound: true
407 : })
408 :
409 0 : const { cameraState, modelTransform } = reported
410 0 : if (!cameraState && !modelTransform) return
411 :
412 0 : const targetGeneration = load3d.currentLoadGeneration
413 0 : void load3d
414 : .whenLoadIdle()
415 : .then(() => {
416 0 : if (load3d.currentLoadGeneration !== targetGeneration) return
417 0 : if (cameraState) load3d.setCameraState(cameraState)
418 0 : if (modelTransform) load3d.applyModelTransform(modelTransform)
419 : })
420 : .catch((error) => {
421 0 : console.error(
422 : `Failed to apply input camera_info / model_3d_info from ${comfyClass}:`,
423 : error
424 : )
425 : })
426 : }
427 :
428 48 : function createPreview3DAdvancedExtension(
429 : comfyClass: string,
430 : extensionName: string,
431 : loadFolder: LoadFolder
432 : ): ComfyExtension {
433 48 : return {
434 : name: extensionName,
435 :
436 : onNodeOutputsUpdated(
437 : nodeOutputs: Record<NodeLocatorId, NodeExecutionOutput>
438 : ) {
439 0 : for (const [locatorId, output] of Object.entries(nodeOutputs)) {
440 0 : const reported = readModel3DOutput(output)
441 0 : if (!reported) continue
442 :
443 0 : const node = getNodeByLocatorId(app.rootGraph, locatorId)
444 0 : if (!node || node.constructor.comfyClass !== comfyClass) continue
445 :
446 0 : useLoad3d(node).waitForLoad3d((load3d) => {
447 0 : applyPreview3DAdvancedResult(
448 : node,
449 : load3d,
450 : reported,
451 : loadFolder,
452 : comfyClass
453 : )
454 : })
455 : }
456 : },
457 :
458 : getNodeMenuItems(node: LGraphNode): (IContextMenuValue | null)[] {
459 24 : if (node.constructor.comfyClass !== comfyClass) return []
460 :
461 0 : return createNodeExportMenuItems(node)
462 : },
463 :
464 : async nodeCreated(node: LGraphNode) {
465 52 : if (node.constructor.comfyClass !== comfyClass) return
466 :
467 0 : const [oldWidth, oldHeight] = node.size
468 :
469 0 : node.setSize([Math.max(oldWidth, 400), Math.max(oldHeight, 550)])
470 :
471 0 : await nextTick()
472 :
473 0 : const onExecuted = node.onExecuted
474 0 : const { onLoad3dReady, waitForLoad3d } = useLoad3d(node)
475 :
476 0 : onLoad3dReady((load3d) => {
477 0 : const lastTimeModelFile = node.properties['Last Time Model File']
478 0 : if (!lastTimeModelFile) return
479 :
480 0 : const lastTimeModelFolder = node.properties['Last Time Model Folder']
481 0 : const folder =
482 : lastTimeModelFolder === 'temp' || lastTimeModelFolder === 'output'
483 : ? lastTimeModelFolder
484 : : loadFolder
485 :
486 : const config = new Load3DConfiguration(load3d, node.properties)
487 0 : config.configureForSaveMesh(folder, lastTimeModelFile as string, {
488 : silentOnNotFound: true
489 : })
490 :
491 : const cameraConfig = node.properties['Camera Config'] as
492 : | CameraConfig
493 : | undefined
494 0 : const cameraState = cameraConfig?.state
495 0 : if (!cameraState) return
496 :
497 0 : const targetGeneration = load3d.currentLoadGeneration
498 0 : void load3d
499 : .whenLoadIdle()
500 : .then(() => {
501 0 : if (load3d.currentLoadGeneration !== targetGeneration) return
502 0 : load3d.setCameraState(cameraState)
503 0 : load3d.forceRender()
504 : })
505 : .catch((error) => {
506 0 : console.error(
507 : `Failed to restore camera state for ${comfyClass}:`,
508 : error
509 : )
510 : })
511 : })
512 :
513 0 : waitForLoad3d((load3d) => {
514 0 : const sceneWidget = node.widgets?.find(
515 : (w) => w.name === 'viewport_state'
516 : )
517 0 : if (!sceneWidget) return
518 :
519 0 : const resolveLoad3d = () => nodeToLoad3dMap.get(node) ?? load3d
520 :
521 0 : const widthWidget = node.widgets?.find((w) => w.name === 'width')
522 0 : const heightWidget = node.widgets?.find((w) => w.name === 'height')
523 0 : if (widthWidget && heightWidget) {
524 0 : load3d.setTargetSize(
525 : widthWidget.value as number,
526 : heightWidget.value as number
527 : )
528 0 : widthWidget.callback = (value: number) => {
529 0 : resolveLoad3d().setTargetSize(value, heightWidget.value as number)
530 : }
531 0 : heightWidget.callback = (value: number) => {
532 0 : resolveLoad3d().setTargetSize(widthWidget.value as number, value)
533 : }
534 : }
535 :
536 0 : sceneWidget.serializeValue = async () => {
537 0 : const currentLoad3d = nodeToLoad3dMap.get(node)
538 0 : if (!currentLoad3d) {
539 0 : console.error('No load3d instance found for node')
540 0 : return null
541 : }
542 :
543 0 : const cameraConfig: CameraConfig = (node.properties[
544 : 'Camera Config'
545 : ] as CameraConfig | undefined) || {
546 : cameraType: currentLoad3d.getCurrentCameraType(),
547 : fov: currentLoad3d.cameraManager.perspectiveCamera.fov
548 : }
549 0 : const cameraState = currentLoad3d.getCameraState()
550 0 : cameraConfig.state = cameraState
551 0 : node.properties['Camera Config'] = cameraConfig
552 :
553 0 : const modelInfo = currentLoad3d.getModelInfo()
554 : const model_3d_info: Model3DInfo = modelInfo ? [modelInfo] : []
555 :
556 0 : return {
557 : image: '',
558 : mask: '',
559 : normal: '',
560 : camera_info: cameraState,
561 : recording: '',
562 : model_3d_info
563 : }
564 : }
565 :
566 0 : node.onExecuted = function (output: NodeExecutionOutput) {
567 0 : onExecuted?.call(this, output)
568 :
569 0 : const reported = readModel3DOutput(output)
570 0 : if (!reported) {
571 0 : const msg = t('toastMessages.unableToGetModelFilePath')
572 0 : console.error(msg)
573 0 : useToast().warning(msg)
574 0 : return
575 : }
576 :
577 0 : applyPreview3DAdvancedResult(
578 : node,
579 : resolveLoad3d(),
580 : reported,
581 : loadFolder,
582 : comfyClass
583 : )
584 : }
585 : })
586 : }
587 : }
588 : }
589 :
590 24 : export const preview3DAdvancedExtension: ComfyExtension =
591 : createPreview3DAdvancedExtension(
592 : 'Preview3DAdvanced',
593 : 'Comfy.Preview3DAdvanced',
594 : 'temp'
595 : )
596 :
597 24 : export const save3DAdvancedExtension: ComfyExtension =
598 : createPreview3DAdvancedExtension(
599 : 'Save3DAdvanced',
600 : 'Comfy.Save3DAdvanced',
601 : 'output'
602 : )
|