Line data Source code
1 : import { once } from 'es-toolkit'
2 :
3 : import { reportError } from '@comfyorg/telemetry/reportError'
4 : import { createUuidv4 } from '@/utils/uuid'
5 :
6 : import { recordDevEvent } from './devPanelLog'
7 : import type { PermanentDocSubscribeRefusalCode } from './docFrameCodes'
8 : import { isPermanentSubscribeRefusalCode } from './docFrameCodes'
9 :
10 : // FE-1902: the doc id is otherwise held only in memory (set on turn ack), so a
11 : // panel remount loses the binding until the NEXT turn ack. Persist it per-tab
12 : // in sessionStorage so an in-page remount can rebind immediately. A full page
13 : // reload deliberately does NOT rebind (see the nonce below): it mints a new
14 : // nonce, refuses the pre-reload record, and waits for the next turn ack.
15 : //
16 : // FEC-5: a bare `docId` string has no owner and no lifetime, so it survives
17 : // (a) a workflow switch in the same browser tab - the NEXT panel mount rebinds
18 : // to whichever workflow last confirmed a subscribe, not necessarily the one
19 : // about to become active - and (b) a browser-tab duplication, which clones
20 : // sessionStorage verbatim into a second tab that never subscribed to that doc
21 : // at all. Neither case can be caught by re-checking `workflowId`, because the
22 : // whole reason a rebind is attempted is that the caller does NOT yet know
23 : // which workflow it's asking about. Instead the persisted record carries (1)
24 : // a per-page-load session nonce, so a value only ever rebinds within the
25 : // SAME top-level navigation that wrote it - a duplicated tab gets a fresh
26 : // nonce and its inherited record is refused - and (2) a short expiry that
27 : // slides while the doc keeps delivering frames, so a tab left idle past the
28 : // window a doc realistically stays relevant is refused rather than trusted
29 : // indefinitely. (1) closes case (b). Case (a) happens inside one page load,
30 : // so the nonce cannot see it; it is only BOUNDED by (2), not closed. The
31 : // `fec-docid-1` reproducer tracks the remaining same-tab window.
32 2355 : const DOC_ID_SESSION_KEY = 'Comfy.Agent.CrdtDocId'
33 2355 : const DOC_ID_TTL_MS = 5 * 60 * 1000
34 : // Re-stamp the expiry on doc traffic at most this often, so a busy channel
35 : // does not turn every frame into a sessionStorage write.
36 2355 : const DOC_ID_REFRESH_INTERVAL_MS = DOC_ID_TTL_MS / 2
37 :
38 : // FE-1901 (poc-2): a `doc_subscribed {ok:false}` is a SERVER refusal - e.g.
39 : // the subscribe raced the doc-host before the turn ack minted the doc. The
40 : // bridge's transport-level reconcile can never repair it: the frame WAS
41 : // delivered, so intent already equals reality. Retry the subscribe itself
42 : // with bounded exponential backoff while the desired doc is unchanged.
43 2355 : const SUBSCRIBE_RETRY_BASE_MS = 500
44 2355 : const SUBSCRIBE_RETRY_MAX_ATTEMPTS = 6
45 :
46 : export type SubscribeGiveUpReason = 'refused' | 'retry_exhausted' | 'timeout'
47 :
48 2355 : const SILENT_PERMANENT_REFUSAL_CODES: ReadonlySet<PermanentDocSubscribeRefusalCode> =
49 : new Set(['unsupported'])
50 :
51 0 : function isNotifyingPermanentRefusalCode(
52 : code: PermanentDocSubscribeRefusalCode
53 : ): boolean {
54 0 : return !SILENT_PERMANENT_REFUSAL_CODES.has(code)
55 : }
56 :
57 : /**
58 : * A `doc_subscribe` that left the transport and was never answered is retried
59 : * on the same lineage after this long. The ingest relay's own resync budget is
60 : * 15 s, so a shorter window would stack a duplicate subscribe behind a slow but
61 : * live catch-up.
62 : */
63 2355 : export const SUBSCRIBE_ACK_TIMEOUT_MS = 15_000
64 : // The third unanswered attempt is terminal: frames at 0 s, 15 s and 30 s,
65 : // give-up at 45 s. Silent attempts also count into the shared refusal budget.
66 : export const SUBSCRIBE_ACK_MAX_TIMEOUTS = 3
67 :
68 : /**
69 : * Recency heartbeat budget (BE-9740's FE half): a bound, healthy channel that
70 : * delivers NO doc-scoped frame for this long gets ONE active probe - a
71 : * resubscribe whose state-vector catch-up is a no-op on a healthy channel and
72 : * exactly the observed recovery on a stale one. A stale channel and an idle
73 : * workflow look identical passively, so expiry probes instead of alarming.
74 : */
75 2355 : export const STALE_AFTER_MS = 30_000
76 :
77 : /**
78 : * PM-1405 RCA: a confirmed subscribe (`doc_subscribed: {ok: true}`) is not
79 : * itself the catch-up - the host sends the ack and the catch-up
80 : * `doc_update` as two separate frames, and the second can be acked-but-never
81 : * sent (an observed real anomaly, not a hypothetical). Relying on the
82 : * passive {@link STALE_AFTER_MS} heartbeat alone to notice leaves the canvas
83 : * with nothing to show for up to 30 real seconds. Probe again this much
84 : * sooner, once per catch-up-gap episode, right after the first confirmed
85 : * subscribe: a resubscribe is a no-op on a doc that had nothing to catch up
86 : * on, and the active repair on one whose catch-up went missing. A backend
87 : * that keeps acknowledging the resubscribe without ever minting content is a
88 : * valid state too, so this fast probe fires at most once per gap - see
89 : * `usedCatchUpGrace`.
90 : */
91 2355 : export const SUBSCRIBE_CATCHUP_GRACE_MS = 2_000
92 :
93 2355 : const pageSessionNonce = once(createUuidv4)
94 :
95 : interface PersistedDocIdRecord {
96 : docId: string
97 : nonce: string
98 : expiresAt: number
99 : }
100 :
101 0 : function safeSessionStorage(): Storage | null {
102 0 : try {
103 0 : return window.sessionStorage
104 : } catch {
105 0 : return null
106 : }
107 : }
108 :
109 0 : function persistDocId(docId: string): void {
110 0 : try {
111 0 : const record: PersistedDocIdRecord = {
112 : docId,
113 : nonce: pageSessionNonce(),
114 : expiresAt: Date.now() + DOC_ID_TTL_MS
115 : }
116 0 : safeSessionStorage()?.setItem(DOC_ID_SESSION_KEY, JSON.stringify(record))
117 : } catch {
118 0 : return
119 : }
120 : }
121 :
122 0 : function readPersistedDocId(): string | null {
123 0 : try {
124 0 : const raw = safeSessionStorage()?.getItem(DOC_ID_SESSION_KEY)
125 0 : if (!raw) return null
126 0 : const record = JSON.parse(raw) as Partial<PersistedDocIdRecord>
127 0 : if (
128 : typeof record.docId !== 'string' ||
129 : record.docId.length === 0 ||
130 : typeof record.nonce !== 'string' ||
131 : typeof record.expiresAt !== 'number'
132 : ) {
133 0 : return null
134 : }
135 0 : if (record.nonce !== pageSessionNonce()) return null
136 0 : if (Date.now() >= record.expiresAt) return null
137 0 : return record.docId
138 : } catch {
139 0 : return null
140 : }
141 : }
142 :
143 0 : function clearPersistedDocId(): void {
144 0 : try {
145 0 : safeSessionStorage()?.removeItem(DOC_ID_SESSION_KEY)
146 : } catch {
147 : // Best-effort.
148 : }
149 : }
150 :
151 2355 : export class AgentCrdtDocLifecycle {
152 : private subscribeRetryTimer: ReturnType<typeof setTimeout> | null = null
153 : private subscribeRetryAttempt = 0
154 : // The recency heartbeat: armed only while a subscribe is CONFIRMED (bound +
155 : // healthy by definition), slid forward by every doc-scoped frame, cancelled
156 : // by the same lifecycle exits as the subscribe retry. The probe is
157 : // `resubscribe()` (not `reconcile()`, which no-ops while intent equals
158 : // reality - and a stale channel's intent DOES equal reality). Within one
159 : // catch-up-gap episode (see `usedCatchUpGrace`), only the first confirmed
160 : // subscribe arms SUBSCRIBE_CATCHUP_GRACE_MS instead of the full budget;
161 : // every later confirm in the same episode, and every re-arm on live
162 : // traffic or on the probe firing, uses the full budget.
163 : private staleProbeTimer: ReturnType<typeof setTimeout> | null = null
164 : // Armed by every subscribe frame that leaves the transport, disarmed by its
165 : // answer (confirm or refusal). Expiry is the third outcome the bridge cannot
166 : // see: the frame was delivered, intent equals reality, and nothing answers.
167 : private ackTimer: ReturnType<typeof setTimeout> | null = null
168 : private ackTimeouts = 0
169 : // Latched after the silent budget is spent so neither the recency probe nor
170 : // a status-frame reconcile can turn the bounded retry into an unbounded one.
171 : // Released only by a lifecycle edge: confirm, reconnect, retarget.
172 : private gaveUp = false
173 : private recoverableGiveUpReason: Extract<
174 : SubscribeGiveUpReason,
175 : 'retry_exhausted' | 'timeout'
176 : > | null = null
177 : private lastReportedRecoverableWorkflowId: string | null = null
178 : // The workflow id last notified (reportError + onGaveUp) for a permanent
179 : // refusal. A reconnect to the SAME workflow answered with the SAME
180 : // permanent code is one unchanged fact, not a new one - re-arm only when
181 : // the target actually changes (retarget/doc_reset) or a subscribe for it
182 : // actually succeeds (`onSubscribeConfirmed`), never on a bare reconnect
183 : // that never recovers. See `resetNotifiedGiveUp`.
184 : private lastNotifiedGiveUpWorkflowId: string | null = null
185 : // Stale-ack race: an ack-timeout resubscribe can leave the OLDER attempt
186 : // still outstanding at the doc-host while a NEWER one (sent on the same
187 : // lineage) confirms first. A confirm and a permanent refusal can never
188 : // both be the answer to the SAME subscribe, so once this is true, any
189 : // permanent refusal that arrives before another subscribe is sent cannot
190 : // be answering the one that just confirmed - it must be the older
191 : // attempt's late answer - and `onSubscribeRefused` discards it instead of
192 : // tearing down the confirm's own probe/latch. Cleared by the next actual
193 : // send (`onSubscribeSent`), which starts a fresh round.
194 : private confirmedSinceLastSend = false
195 : // FEC-5: `Date.now()` of the last persisted-record write by this instance.
196 : // A confirmed subscribe always writes; doc-scoped frames re-stamp the expiry
197 : // no more often than DOC_ID_REFRESH_INTERVAL_MS, so a doc that keeps
198 : // delivering frames keeps its rebind window instead of lapsing mid-session.
199 : private lastPersistedAt = 0
200 : // PM-1405: caps the fast catch-up probe to one shot per gap episode. Set on
201 : // the first confirmed subscribe of an episode, cleared when real content
202 : // arrives (the episode is over) or the episode restarts (bind/reconnect).
203 : private usedCatchUpGrace = false
204 :
205 : constructor(
206 1 : private readonly workflowId: () => string | null,
207 1 : private readonly resubscribe: () => boolean | void,
208 1 : private readonly onGaveUp: (reason: SubscribeGiveUpReason) => void
209 : ) {}
210 :
211 : readPersistedDocId(): string | null {
212 0 : return readPersistedDocId()
213 : }
214 :
215 : clearPersistedDocId(): void {
216 0 : clearPersistedDocId()
217 : }
218 :
219 : onSubscribeConfirmed(): void {
220 0 : this.clearAckTimer()
221 0 : this.gaveUp = false
222 0 : this.recoverableGiveUpReason = null
223 0 : this.confirmedSinceLastSend = true
224 0 : this.resetNotifiedGiveUp()
225 0 : this.clearSubscribeRetry()
226 0 : if (this.usedCatchUpGrace) {
227 0 : this.armStaleProbe()
228 : } else {
229 0 : this.usedCatchUpGrace = true
230 0 : this.armStaleProbe(SUBSCRIBE_CATCHUP_GRACE_MS)
231 : }
232 0 : const workflowId = this.workflowId()
233 0 : if (workflowId !== null) this.persistConfirmedDocId(workflowId)
234 : }
235 :
236 : /** Returns whether this refusal newly triggered a give-up notification. */
237 : onSubscribeRefused(code?: string): boolean {
238 0 : if (code !== undefined && isPermanentSubscribeRefusalCode(code)) {
239 0 : if (this.confirmedSinceLastSend) return false
240 0 : const supersedesRecoverableGiveUp = this.recoverableGiveUpReason !== null
241 0 : this.recoverableGiveUpReason = null
242 0 : if (supersedesRecoverableGiveUp) this.gaveUp = false
243 0 : this.clearAckTimer()
244 0 : this.clearStaleProbe()
245 0 : return this.giveUpPermanently(code, isNotifyingPermanentRefusalCode(code))
246 : }
247 0 : this.clearAckTimer()
248 0 : this.clearStaleProbe()
249 0 : this.scheduleSubscribeRetry()
250 0 : return false
251 : }
252 :
253 : refusalIsFinal(): boolean {
254 0 : return !this.confirmedSinceLastSend && this.subscribeRetryTimer === null
255 : }
256 :
257 : stopProbing(): void {
258 0 : this.clearAckTimer()
259 0 : this.clearSubscribeRetry()
260 0 : this.clearStaleProbe()
261 0 : this.gaveUp = true
262 0 : this.recoverableGiveUpReason = null
263 : }
264 :
265 : onSubscribeSent(workflowId: string): void {
266 0 : this.clearAckTimer()
267 0 : this.confirmedSinceLastSend = false
268 0 : if (this.gaveUp) return
269 0 : this.ackTimer = setTimeout(() => {
270 0 : this.ackTimer = null
271 0 : if (this.workflowId() !== workflowId) return
272 0 : this.ackTimeouts += 1
273 0 : if (
274 : this.ackTimeouts >= SUBSCRIBE_ACK_MAX_TIMEOUTS ||
275 : this.subscribeRetryAttempt >= SUBSCRIBE_RETRY_MAX_ATTEMPTS
276 : ) {
277 0 : this.giveUp(
278 : workflowId,
279 : this.subscribeRetryAttempt >= SUBSCRIBE_RETRY_MAX_ATTEMPTS
280 : ? 'retry_exhausted'
281 : : 'timeout'
282 : )
283 0 : return
284 : }
285 0 : this.subscribeRetryAttempt += 1
286 0 : recordDevEvent('subscribe_ack_timeout', {
287 : attempt: this.subscribeRetryAttempt,
288 : workflowId
289 : })
290 0 : this.resubscribeOrScheduleRetry()
291 : }, SUBSCRIBE_ACK_TIMEOUT_MS)
292 : }
293 :
294 : /**
295 : * A new socket is a new server-side session: every budget starts over,
296 : * including the permanent-give-up latch (a redeployed host may now be
297 : * readable). `lastNotifiedGiveUpWorkflowId` deliberately survives this -
298 : * see `resetNotifiedGiveUp` - so a refusal that recurs for the same
299 : * workflow after reconnecting doesn't re-fire the notification.
300 : */
301 : onReconnected(): void {
302 1 : this.resetForNewAttempt()
303 : }
304 :
305 : onDocumentUpdate(): void {
306 0 : this.usedCatchUpGrace = false
307 0 : if (this.staleProbeTimer !== null) this.armStaleProbe()
308 0 : this.refreshPersistedDocId()
309 : }
310 :
311 : onDocumentResult(): void {
312 0 : this.usedCatchUpGrace = false
313 0 : if (this.staleProbeTimer === null) return
314 0 : this.armStaleProbe()
315 0 : this.refreshPersistedDocId()
316 : }
317 :
318 : clearStaleProbe(): void {
319 2 : if (this.staleProbeTimer !== null) {
320 0 : clearTimeout(this.staleProbeTimer)
321 0 : this.staleProbeTimer = null
322 : }
323 : }
324 :
325 : private shouldDeferSubscribe(): boolean {
326 0 : return this.gaveUp || this.subscribeRetryTimer !== null
327 : }
328 :
329 : clearForRetarget(): void {
330 1 : this.resetForNewAttempt()
331 1 : this.resetNotifiedGiveUp()
332 : }
333 :
334 : /** Re-reads the bound document, deferring to a retry or give-up in progress. */
335 : refresh(): void {
336 0 : if (this.shouldDeferSubscribe()) return
337 0 : this.resubscribeOrScheduleRetry()
338 : }
339 :
340 : retry(): boolean {
341 0 : const reason = this.recoverableGiveUpReason
342 0 : if (reason === null) return false
343 0 : this.resetForNewAttempt()
344 0 : if (this.resubscribe() === true) return true
345 0 : this.gaveUp = true
346 0 : this.recoverableGiveUpReason = reason
347 0 : return false
348 : }
349 :
350 : /**
351 : * Lets a permanent refusal for a different episode of the same workflow
352 : * id notify again. Called when the subscribed target actually changes
353 : * (retarget), its lineage restarts (`doc_reset`), or it actually confirms
354 : * (`onSubscribeConfirmed`) - never on a bare reconnect that never
355 : * recovers, which keeps the same episode and the same unrecoverable fact.
356 : */
357 : resetNotifiedGiveUp(): void {
358 1 : this.lastNotifiedGiveUpWorkflowId = null
359 1 : this.lastReportedRecoverableWorkflowId = null
360 : }
361 :
362 : destroy(): void {
363 0 : this.clearForRetarget()
364 : }
365 :
366 : private persistConfirmedDocId(docId: string): void {
367 0 : persistDocId(docId)
368 0 : this.lastPersistedAt = Date.now()
369 : }
370 :
371 : private refreshPersistedDocId(): void {
372 0 : const workflowId = this.workflowId()
373 0 : if (workflowId === null) return
374 0 : if (Date.now() - this.lastPersistedAt < DOC_ID_REFRESH_INTERVAL_MS) return
375 0 : this.persistConfirmedDocId(workflowId)
376 : }
377 :
378 : private armStaleProbe(delayMs: number = STALE_AFTER_MS): void {
379 0 : this.clearStaleProbe()
380 0 : const isCatchUpProbe = delayMs !== STALE_AFTER_MS
381 0 : this.staleProbeTimer = setTimeout(() => {
382 0 : this.staleProbeTimer = null
383 0 : if (this.gaveUp) return
384 0 : this.armStaleProbe()
385 0 : if (this.ackTimer !== null) return
386 0 : recordDevEvent(isCatchUpProbe ? 'catchup_probe' : 'stale_probe', {
387 : workflowId: this.workflowId()
388 : })
389 0 : this.resubscribeOrScheduleRetry()
390 : }, delayMs)
391 : }
392 :
393 : private resetForNewAttempt(): void {
394 2 : this.clearAckTimer()
395 2 : this.clearSubscribeRetry()
396 2 : this.clearStaleProbe()
397 2 : this.gaveUp = false
398 2 : this.recoverableGiveUpReason = null
399 2 : this.usedCatchUpGrace = false
400 : }
401 :
402 : private clearAckTimer(): void {
403 2 : if (this.ackTimer !== null) {
404 0 : clearTimeout(this.ackTimer)
405 0 : this.ackTimer = null
406 : }
407 : }
408 :
409 : /** Returns whether this call actually notified (reportError + onGaveUp). */
410 : private giveUpPermanently(code: string, notify: boolean): boolean {
411 : // A redelivered/duplicate refusal frame answers nothing new once the
412 : // latch is already set — without this guard it would re-run the report
413 : // and the caller's `onGaveUp` (toast/telemetry) a second time.
414 0 : if (this.gaveUp) return false
415 0 : this.gaveUp = true
416 : // An already-armed retry from an earlier, retryable refusal must not
417 : // survive the permanent latch: its callback checks only `workflowId()`,
418 : // not `gaveUp`, so left alone it would still fire a resubscribe later.
419 0 : this.clearSubscribeRetry()
420 0 : if (!notify) return false
421 0 : const workflowId = this.workflowId()
422 : // A reconnect that lands the same permanent code for the same workflow
423 : // is one unchanged fact - only a different workflow (retarget/doc_reset,
424 : // see `resetNotifiedGiveUp`) is a new one worth telling the person about.
425 0 : if (workflowId !== null && workflowId === this.lastNotifiedGiveUpWorkflowId)
426 0 : return false
427 0 : this.lastNotifiedGiveUpWorkflowId = workflowId
428 0 : recordDevEvent(
429 : 'subscribe_refused_permanent',
430 : { code, terminal: true },
431 : { level: 'warn' }
432 : )
433 0 : reportError(new Error(`agent doc subscribe permanently refused: ${code}`), {
434 : surface: 'agent',
435 : errorType: 'failure_confirming_agent_doc_subscribe',
436 : level: 'warning',
437 : tags: { feature_area: 'agent', operation: 'sync', outcome: 'gave_up' }
438 : })
439 0 : this.onGaveUp('refused')
440 0 : return true
441 : }
442 :
443 : private giveUp(
444 : workflowId: string,
445 : reason: Extract<SubscribeGiveUpReason, 'retry_exhausted' | 'timeout'>
446 : ): void {
447 0 : this.clearStaleProbe()
448 0 : this.gaveUp = true
449 0 : this.recoverableGiveUpReason = reason
450 0 : recordDevEvent(
451 : reason === 'timeout' ? 'subscribe_ack_timeout' : 'subscribe_retry',
452 : { attempt: this.subscribeRetryAttempt, workflowId, terminal: true },
453 : { level: 'warn' }
454 : )
455 0 : if (workflowId !== this.lastReportedRecoverableWorkflowId) {
456 0 : this.lastReportedRecoverableWorkflowId = workflowId
457 0 : reportError(
458 : new Error(
459 : reason === 'timeout'
460 : ? 'agent doc subscribe was sent but never acknowledged'
461 : : 'agent doc subscription retries were exhausted'
462 : ),
463 : {
464 : surface: 'agent',
465 : errorType: 'failure_confirming_agent_doc_subscribe',
466 : level: 'warning',
467 : tags: { feature_area: 'agent', operation: 'sync', outcome: 'gave_up' }
468 : }
469 : )
470 : }
471 0 : this.onGaveUp(reason)
472 : }
473 :
474 : private clearSubscribeRetry(): void {
475 2 : if (this.subscribeRetryTimer !== null) {
476 0 : clearTimeout(this.subscribeRetryTimer)
477 0 : this.subscribeRetryTimer = null
478 : }
479 2 : this.subscribeRetryAttempt = 0
480 2 : this.ackTimeouts = 0
481 : }
482 :
483 : private scheduleSubscribeRetry(): void {
484 0 : if (this.shouldDeferSubscribe()) return
485 0 : const target = this.workflowId()
486 0 : if (target === null) return
487 0 : if (this.subscribeRetryAttempt >= SUBSCRIBE_RETRY_MAX_ATTEMPTS) {
488 0 : this.giveUp(target, 'retry_exhausted')
489 0 : return
490 : }
491 0 : const delay = SUBSCRIBE_RETRY_BASE_MS * 2 ** this.subscribeRetryAttempt
492 0 : this.subscribeRetryAttempt += 1
493 0 : this.subscribeRetryTimer = setTimeout(() => {
494 0 : this.subscribeRetryTimer = null
495 0 : if (this.workflowId() !== target) return
496 0 : recordDevEvent('subscribe_retry', {
497 : attempt: this.subscribeRetryAttempt,
498 : workflowId: target
499 : })
500 0 : this.resubscribeOrScheduleRetry()
501 : }, delay)
502 : }
503 :
504 : private resubscribeOrScheduleRetry(): void {
505 0 : if (this.resubscribe() !== false) return
506 0 : this.clearStaleProbe()
507 0 : this.scheduleSubscribeRetry()
508 : }
509 : }
|