Line data Source code
1 : /**
2 : * The human write leg's sender (plan 3.2/3.3): takes semantic
3 : * {@link GraphOperation}s from the mint ports, mints wire identity ONCE, and
4 : * drives `doc_ops` frames through the doc frame client with bounded retries
5 : * that never re-mint an `op_id` (changed-payload reuse rejects host-side;
6 : * an unchanged resend converges through the applier's idempotency gate).
7 : *
8 : * Batches are strictly serialized: one in-flight batch at a time, FIFO, so
9 : * op order on the wire matches mint order. `base_version` is read at mint
10 : * time from the follower's last observed sequence.
11 : */
12 : import type { Op } from '@comfyorg/comfy-multi-player'
13 :
14 : import { reportError } from '@comfyorg/telemetry/reportError'
15 :
16 : import type { DocOp, DocOpsResult } from './docFrameClient'
17 : import type { GraphOperation } from './graphOperations'
18 : import {
19 : chunkWireOps,
20 : measureWireOp,
21 : mintWireOps,
22 : WIRE_MAX_OPS_PER_BATCH
23 : } from './opEnvelope'
24 : import type { SizedOp } from './opEnvelope'
25 : import { reportLostEdit } from './reportLostEdit'
26 : import type { UnboundAbortReason } from './reportLostEdit'
27 :
28 1 : const SEND_RETRY_LIMIT = 5
29 1 : const SEND_RETRY_INTERVAL_MS = 500
30 1 : const RESULT_TIMEOUT_MS = 10_000
31 1 : const FAILURE_REPORT_WINDOW_MS = 60_000
32 1 : const MAX_FAILURE_REPORTS_PER_WINDOW = 3
33 :
34 : /**
35 : * Admits at most {@link MAX_FAILURE_REPORTS_PER_WINDOW} reports per
36 : * {@link FAILURE_REPORT_WINDOW_MS}; the window re-arms by time, not by success.
37 : */
38 5 : export function createReportBudget(): () => boolean {
39 5 : let reports = 0
40 5 : let windowStartedAt: number | null = null
41 5 : return () => {
42 0 : const now = performance.now()
43 0 : if (
44 : windowStartedAt === null ||
45 : now - windowStartedAt >= FAILURE_REPORT_WINDOW_MS
46 : ) {
47 0 : reports = 0
48 0 : windowStartedAt = now
49 : }
50 0 : if (reports >= MAX_FAILURE_REPORTS_PER_WINDOW) return false
51 0 : reports++
52 0 : return true
53 : }
54 : }
55 :
56 : /**
57 : * The sender's view of a parsed `doc_ops_result`. Derived from the
58 : * authoritative {@link DocOpsResult} rather than restated, so the fields the
59 : * sender consumes retain their canonical names and types. `workflowId` is
60 : * optional only because a sender may be driven without one.
61 : */
62 : export type OpsResultView = Pick<
63 : DocOpsResult,
64 : 'ok' | 'seq' | 'applied' | 'skipped' | 'lwwDropped' | 'code' | 'failed'
65 : > &
66 : Partial<Pick<DocOpsResult, 'workflowId'>>
67 :
68 : export interface OpSenderDeps {
69 : /**
70 : * `DocFrameClient.sendOps` shape: false = the transport cannot carry it
71 : * now. Receives the wire objects measured at admission, never the ops.
72 : */
73 : sendOps(workflowId: string, tab: string, ops: readonly DocOp[]): boolean
74 : /** Subscribe to `doc_ops_result` frames; returns unsubscribe. */
75 : onOpsResult(listener: (result: OpsResultView) => void): () => void
76 : /**
77 : * The bound workflow id, or null when no doc is bound. Read at mint time to
78 : * address the batch and re-read before EVERY send and resend: a batch whose
79 : * workflow is no longer bound (subscription refused, tab moved to another
80 : * doc) is never carried or re-addressed and settles at once, 'unconfirmed'
81 : * if the transport already carried it and 'undeliverable' otherwise.
82 : */
83 : workflowId(): string | null
84 : tab: string
85 : /** `human:<user>:<tab>` (vocabulary §7). */
86 : actor(): string
87 : /** The follower's last observed doc sequence (a floor for `base_version`). */
88 : baseVersion(): number
89 : /** The highest Lamport counter in the document the follower holds. */
90 : observedCounter(): number | undefined
91 : onBatchSettled(
92 : outcome: BatchOutcome,
93 : admissionWorkflowId: string | null
94 : ): void
95 : }
96 :
97 : export type BatchOutcome =
98 : | {
99 : state: 'acknowledged'
100 : ops: Op[]
101 : result: OpsResultView
102 : /** The batch was sent again, so a skipped op may be a lost first send. */
103 : resent?: boolean
104 : }
105 : | { state: 'unacknowledged'; ops: Op[] }
106 : | { state: 'unconfirmed'; ops: Op[] }
107 : | { state: 'undeliverable'; ops: Op[] }
108 :
109 : interface AdmissionGroup {
110 : workflowId: string
111 : ops: Op[]
112 : }
113 :
114 : export interface OpSender {
115 : enqueue(operations: GraphOperation[]): void
116 : /**
117 : * Mint and target-pin operations into the open admission group without
118 : * starting transport delivery. Consecutive admissions for one workflow
119 : * share the group until `flush()` seals it.
120 : *
121 : * Each op is serialized once here; a malformed op settles `undeliverable`
122 : * at once and the rest of the admission is still admitted. Mint order is
123 : * preserved per document, not across documents, including when a
124 : * serializer or settlement listener re-enters `admit()` or `enqueue()`:
125 : * the re-entrant admission lands behind the one in progress, and a flush
126 : * requested meanwhile runs once every pending admission is committed.
127 : */
128 : admit(operations: GraphOperation[]): void
129 : /** Seal the open admission group into wire batches and start delivery. */
130 : flush(): void
131 : /** Unsettled batch count for observability; 0 = drained. */
132 : pending(): number
133 : /** Every unsettled batch, in-flight first, each addressed to its mint-time workflow. */
134 : pendingOps(): ReadonlyArray<AdmissionGroup>
135 : /**
136 : * The bound workflow's tab went inactive: the subscription is paused, not
137 : * lost. Until `resume()`, a batch reaching `transmit()` is parked instead
138 : * of sent or settled, so it neither reaches an unsubscribed doc nor dies
139 : * `undeliverable`. A batch already sent keeps its result timer, so a late
140 : * result still settles it. Idempotent.
141 : */
142 : suspend(): void
143 : /**
144 : * Re-transmit the parked batch, if any. The transmit-time binding check
145 : * still runs, so a batch whose workflow is no longer bound settles
146 : * `undeliverable` exactly as it would have. Idempotent.
147 : */
148 : resume(): void
149 : /** Retries a batch held after transport exhaustion. */
150 : retryTransport(): void
151 : /**
152 : * Eager abort seam (FE #16637 residual): settle the in-flight batch NOW
153 : * ('unconfirmed' once transmitted, 'undeliverable' otherwise) if its
154 : * mint-time workflow no longer matches `deps.workflowId()`, instead of
155 : * waiting out the 10 s result-silence window before the next transmit
156 : * re-reads it. A caller with an earlier
157 : * signal that the subscription is gone (e.g. `doc_subscribed {ok:false}`)
158 : * should call this immediately; a no-op otherwise (still bound, or the
159 : * unbind already resolved through the normal transmit-time check).
160 : *
161 : * Not for reconnect: `resubscribe()` re-binds the SAME id synchronously on
162 : * a live socket, so this stays a no-op there by design — the batch rides
163 : * the result timer to its idempotent resend, which is the right outcome
164 : * (the ops may well have landed), not `undeliverable`.
165 : *
166 : * `reason`, when given, is carried into the lost-edit report so a refused
167 : * resubscribe and a tab-switch retarget are distinguishable instead of
168 : * both collapsing into the bare `'unbound'` code.
169 : */
170 : abortIfUnbound(reason?: UnboundAbortReason): void
171 : /**
172 : * Lineage-break seam: settle the in-flight batch ('unconfirmed' once
173 : * transmitted, 'undeliverable' otherwise) and every queued batch
174 : * `undeliverable` NOW, in mint order, although the doc is still bound. A
175 : * `doc_reset` replaced the document these ops were minted against; the
176 : * human-authored draft that caused it already carries their effect, so
177 : * re-addressing them to the new lineage would apply them twice.
178 : */
179 : abortAll(): void
180 : /** Settle outstanding work, unsubscribe, and permanently stop this sender. */
181 : detach(): void
182 : }
183 :
184 : interface WireBatch {
185 : workflowId: string
186 : ops: Op[]
187 : wire: DocOp[]
188 : }
189 :
190 0 : function liftCounter(op: Op, shift: number): Op {
191 0 : const counter = op.base_version + shift
192 0 : return { ...op, base_version: counter, stamp: [counter, op.actor] }
193 : }
194 :
195 0 : function toWireBatch(workflowId: string, chunk: readonly SizedOp[]): WireBatch {
196 0 : return {
197 : workflowId,
198 : ops: chunk.map((sized) => sized.op),
199 : wire: chunk.map((sized) => sized.wire)
200 : }
201 : }
202 :
203 : /** One admit() call after measurement, awaiting commitment in mint order. */
204 : interface Admission {
205 : workflowId: string | null
206 : minted: Op[]
207 : admitted: SizedOp[]
208 : rejected: Op[]
209 : cause: unknown
210 : exhausted: boolean
211 : }
212 :
213 : interface InFlight extends WireBatch {
214 : opIds: Set<string>
215 : sends: number
216 : reportedThrow: boolean
217 : resent: boolean
218 : parked: boolean
219 : awaitingTransport: boolean
220 : sendMayHaveLeft: boolean
221 : timer: ReturnType<typeof setTimeout> | null
222 : }
223 :
224 1 : export function createOpSender(deps: OpSenderDeps): OpSender {
225 1 : const queue: WireBatch[] = []
226 1 : let queueHead = 0
227 1 : let open: { workflowId: string; ops: SizedOp[] } | null = null
228 1 : let inFlight: InFlight | null = null
229 : // One Lamport cursor per workflow, cleared only by that doc's reset: the
230 : // observed sequence reads 0 between a subscribe and its ack, so only the
231 : // cursor keeps a re-bound workflow's stamps past ones this actor used there.
232 1 : const lastMintedVersions = new Map<string, number>()
233 1 : const lastSentVersions = new Map<string, number>()
234 1 : let detached = false
235 1 : let suspended = false
236 1 : let pumping = false
237 : // Late-result credits: every send a batch leaves the client with may still
238 : // draw a result, including the send whose silence provoked the resend and
239 : // the sends of a batch that has already settled. As ANONYMOUS failures
240 : // (empty id lists, no failure op_id) those are indistinguishable from the
241 : // current batch's, so one is swallowed per credit rather than risk a
242 : // mis-attributed settle, which poisons everything downstream of this seam.
243 : // The cost is not one cycle: a credit reserved for a result that never
244 : // arrives is never drained, so the next batch's own answer pays for it and
245 : // reserves another on its resend. Once one result is lost, every later
246 : // anonymous failure costs a RESULT_TIMEOUT_MS resend, until the host can
247 : // correlate a result to the send it answers.
248 1 : let staleAnonymousBudget = 0
249 1 : const retiredOpIds = new Set<string>()
250 1 : const mayReportLostEdit = createReportBudget()
251 1 : const mayReportSettlementFailure = createReportBudget()
252 1 : const mayReportSerializationFailure = createReportBudget()
253 1 : const mayReportCounterExhaustion = createReportBudget()
254 :
255 0 : function observedFloor(): number {
256 0 : const floor = (deps.observedCounter() ?? -1) + 1
257 0 : return Number.isSafeInteger(floor) ? floor : 0
258 : }
259 :
260 0 : function mintBaseVersion(workflowId: string | null): number {
261 0 : const observed = Math.max(deps.baseVersion(), observedFloor())
262 0 : if (workflowId === null) return observed
263 0 : const lastMinted = lastMintedVersions.get(workflowId)
264 0 : return lastMinted === undefined
265 : ? observed
266 : : Math.max(observed, lastMinted + 1)
267 : }
268 :
269 0 : function retire(batch: InFlight, answered: number): void {
270 0 : const outstanding = batch.sends - answered
271 0 : if (outstanding <= 0) return
272 0 : staleAnonymousBudget += outstanding
273 0 : for (const opId of batch.opIds) retiredOpIds.add(opId)
274 : }
275 :
276 0 : function drainStaleCredit(): void {
277 0 : if (staleAnonymousBudget === 0) return
278 0 : staleAnonymousBudget--
279 0 : if (staleAnonymousBudget === 0) retiredOpIds.clear()
280 : }
281 :
282 0 : function reportDegraded(cause: unknown, errorType: string): void {
283 0 : reportError(cause, {
284 : errorType,
285 : surface: 'agent',
286 : tags: { feature_area: 'agent', operation: 'sync', outcome: 'degraded' }
287 : })
288 : }
289 :
290 1 : function guardedSettlementNotifier(
291 : errorType: string,
292 : code: string
293 : ): (
294 : outcome: BatchOutcome,
295 : workflowId: string | null,
296 : reason?: UnboundAbortReason
297 : ) => void {
298 1 : let reportedFailure = false
299 1 : return (outcome, workflowId, reason) => {
300 0 : if (outcome.state !== 'acknowledged')
301 0 : reportLostEdit(
302 : outcome.state,
303 : outcome.ops,
304 : {
305 : code,
306 : workflowId: workflowId ?? undefined,
307 : reason
308 : },
309 : { shouldReport: mayReportLostEdit() }
310 : )
311 0 : try {
312 0 : deps.onBatchSettled(outcome, workflowId)
313 : } catch (cause) {
314 0 : if (reportedFailure || !mayReportSettlementFailure()) return
315 0 : reportedFailure = true
316 0 : reportDegraded(cause, errorType)
317 : }
318 : }
319 : }
320 :
321 1 : const notifyDetachSettlement = guardedSettlementNotifier(
322 : 'failure_settling_agent_op_sender_detach',
323 : 'detached'
324 : )
325 0 : function settle(
326 : outcome: BatchOutcome,
327 : code: string,
328 : reason?: UnboundAbortReason
329 : ): void {
330 0 : const workflowId = inFlight?.workflowId ?? null
331 0 : if (inFlight?.timer) clearTimeout(inFlight.timer)
332 0 : inFlight = null
333 0 : guardedSettlementNotifier('failure_settling_agent_op_sender', code)(
334 : outcome,
335 : workflowId,
336 : reason
337 : )
338 0 : pump()
339 : }
340 :
341 0 : function shouldPark(batch: InFlight): boolean {
342 0 : return (
343 : suspended &&
344 : (pendingAbortReason?.workflowId !== batch.workflowId ||
345 : deps.workflowId() === batch.workflowId)
346 : )
347 : }
348 :
349 0 : function transmit(batch: InFlight, attempt: number): void {
350 0 : if (detached || inFlight !== batch) return
351 0 : if (shouldPark(batch)) {
352 0 : batch.parked = true
353 0 : return
354 : }
355 0 : if (deps.workflowId() !== batch.workflowId) {
356 0 : settleUnbound(
357 : batch,
358 : 'unbound',
359 : pendingAbortReason?.workflowId === batch.workflowId
360 : ? pendingAbortReason.reason
361 : : undefined
362 : )
363 0 : return
364 : }
365 0 : if (!trySend(batch)) {
366 0 : if (attempt < SEND_RETRY_LIMIT) {
367 0 : batch.timer = setTimeout(
368 : () => transmit(batch, attempt + 1),
369 : SEND_RETRY_INTERVAL_MS
370 : )
371 : } else {
372 0 : batch.awaitingTransport = true
373 : }
374 0 : return
375 : }
376 0 : batch.awaitingTransport = false
377 0 : recordSentCounters(batch)
378 0 : batch.sends++
379 0 : armResultTimeout(batch)
380 : }
381 :
382 0 : function stampAboveObserved(batch: InFlight): void {
383 0 : if (batch.sends > 0 || batch.sendMayHaveLeft) return
384 0 : const counters = batch.ops.map((op) => op.base_version)
385 : const floor = Math.max(
386 : observedFloor(),
387 : (lastSentVersions.get(batch.workflowId) ?? -1) + 1
388 : )
389 0 : const shift = floor - Math.min(...counters)
390 0 : if (shift > 0 && Number.isSafeInteger(Math.max(...counters) + shift)) {
391 0 : batch.ops = batch.ops.map((op) => liftCounter(op, shift))
392 0 : batch.wire = batch.wire.map((wire, index) => ({
393 : ...wire,
394 : base_version: batch.ops[index].base_version,
395 : stamp: batch.ops[index].stamp
396 : }))
397 : }
398 : }
399 :
400 0 : function recordSentCounters(batch: InFlight): void {
401 0 : const highest = Math.max(...batch.ops.map((op) => op.base_version))
402 0 : const lastSent = lastSentVersions.get(batch.workflowId) ?? -1
403 0 : lastSentVersions.set(batch.workflowId, Math.max(lastSent, highest))
404 0 : const lastMinted = lastMintedVersions.get(batch.workflowId) ?? -1
405 0 : lastMintedVersions.set(batch.workflowId, Math.max(lastMinted, highest))
406 : }
407 :
408 0 : function trySend(batch: InFlight): boolean {
409 0 : stampAboveObserved(batch)
410 0 : try {
411 0 : return deps.sendOps(batch.workflowId, deps.tab, batch.wire)
412 : } catch (error) {
413 0 : batch.sendMayHaveLeft = true
414 0 : recordSentCounters(batch)
415 0 : if (!batch.reportedThrow) {
416 0 : batch.reportedThrow = true
417 0 : reportError(error, {
418 : surface: 'agent',
419 : errorType: 'failure_sending_agent_human_ops',
420 : tags: {
421 : failure_kind: 'caught_unexpected',
422 : feature_area: 'agent',
423 : operation: 'sync',
424 : outcome: 'recovered'
425 : },
426 : level: 'error'
427 : })
428 : }
429 0 : return false
430 : }
431 : }
432 :
433 1 : let pendingAbortReason: {
434 : workflowId: string
435 : reason: UnboundAbortReason | undefined
436 : } | null = null
437 :
438 0 : function settleUnbound(
439 : batch: InFlight,
440 : code: string,
441 : reason?: UnboundAbortReason
442 : ): void {
443 0 : if (inFlight !== batch) return
444 0 : retire(batch, 0)
445 0 : settle(
446 : {
447 : state: batch.sends > 0 ? 'unconfirmed' : 'undeliverable',
448 : ops: batch.ops
449 : },
450 : code,
451 : reason
452 : )
453 : }
454 :
455 0 : function armResultTimeout(batch: InFlight): void {
456 0 : if (inFlight !== batch) return
457 0 : batch.timer = setTimeout(() => {
458 0 : if (inFlight !== batch) return
459 0 : if (batch.resent) {
460 0 : retire(batch, 0)
461 0 : settle({ state: 'unacknowledged', ops: batch.ops }, 'result_timeout')
462 0 : return
463 : }
464 0 : batch.resent = true
465 0 : batch.reportedThrow = false
466 0 : transmit(batch, 0)
467 : }, RESULT_TIMEOUT_MS)
468 : }
469 :
470 0 : function drainOutstanding(
471 : notify: (outcome: BatchOutcome, workflowId: string | null) => void
472 : ): void {
473 0 : const unsealed = open
474 0 : open = null
475 0 : const queued = queue.slice(queueHead)
476 0 : queue.length = 0
477 0 : queueHead = 0
478 0 : const active = inFlight
479 0 : inFlight = null
480 0 : if (active) {
481 0 : const batch = active
482 0 : if (batch.timer) clearTimeout(batch.timer)
483 0 : retire(batch, 0)
484 0 : notify(
485 : {
486 : state: batch.sends > 0 ? 'unconfirmed' : 'undeliverable',
487 : ops: batch.ops
488 : },
489 : batch.workflowId
490 : )
491 : }
492 0 : for (const batch of queued) {
493 0 : notify({ state: 'undeliverable', ops: batch.ops }, batch.workflowId)
494 : }
495 0 : if (unsealed) {
496 0 : for (const chunk of chunkWireOps(unsealed.ops)) {
497 0 : notify(
498 : { state: 'undeliverable', ops: chunk.map((sized) => sized.op) },
499 : unsealed.workflowId
500 : )
501 : }
502 : }
503 : }
504 :
505 1 : function dequeue(): WireBatch | undefined {
506 1 : if (queueHead >= queue.length) return
507 0 : const queued = queue[queueHead]
508 0 : queueHead++
509 1 : if (queueHead >= 1_024 && queueHead * 2 >= queue.length) {
510 0 : queue.splice(0, queueHead)
511 0 : queueHead = 0
512 : }
513 0 : return queued
514 : }
515 :
516 1 : function pump(): void {
517 1 : if (pumping) return
518 1 : pumping = true
519 1 : try {
520 1 : while (!detached && inFlight === null) {
521 1 : const queued = dequeue()
522 1 : if (!queued) return
523 0 : inFlight = {
524 : ...queued,
525 : opIds: new Set(queued.ops.map((op) => op.op_id)),
526 : sends: 0,
527 : reportedThrow: false,
528 : resent: false,
529 : parked: false,
530 : awaitingTransport: false,
531 : sendMayHaveLeft: false,
532 : timer: null
533 : }
534 0 : transmit(inFlight, 0)
535 : }
536 : } finally {
537 1 : pumping = false
538 : }
539 : }
540 :
541 0 : function settleUndeliverableInBoundedGroups(
542 : ops: Op[],
543 : notify: (outcome: BatchOutcome, workflowId: string | null) => void,
544 : workflowId: string | null
545 : ): void {
546 0 : for (let index = 0; index < ops.length; index += WIRE_MAX_OPS_PER_BATCH) {
547 0 : notify(
548 : {
549 : state: 'undeliverable',
550 : ops: ops.slice(index, index + WIRE_MAX_OPS_PER_BATCH)
551 : },
552 : workflowId
553 : )
554 : }
555 : }
556 :
557 0 : function measureMintedOps(minted: Op[]): {
558 : admitted: SizedOp[]
559 : rejected: Op[]
560 : cause: unknown
561 : } {
562 0 : const measured = minted.map(measureWireOp)
563 0 : const rejections = measured.filter((result) => !result.admitted)
564 0 : return {
565 : admitted: measured.flatMap((result) =>
566 : result.admitted ? [result.sized] : []
567 : ),
568 : rejected: rejections.map((rejection) => rejection.op),
569 : cause: rejections[0]?.cause
570 : }
571 : }
572 :
573 0 : function mintAdmission(
574 : operations: GraphOperation[],
575 : workflowId: string | null
576 : ): { minted: Op[]; exhausted: boolean } {
577 0 : const baseVersion = mintBaseVersion(workflowId)
578 0 : const actor = deps.actor()
579 0 : const highest = baseVersion + (operations.length - 1)
580 0 : if (!Number.isSafeInteger(highest))
581 0 : return {
582 : minted: mintWireOps(operations, {
583 : actor,
584 : baseVersion: Number.MAX_SAFE_INTEGER
585 : }),
586 : exhausted: true
587 : }
588 0 : const minted = operations.flatMap((operation, index) =>
589 : mintWireOps([operation], { actor, baseVersion: baseVersion + index })
590 : )
591 0 : if (workflowId !== null) lastMintedVersions.set(workflowId, highest)
592 0 : return { minted, exhausted: false }
593 : }
594 :
595 0 : function appendToOpenGroup(workflowId: string, admitted: SizedOp[]): void {
596 0 : if (open?.workflowId !== workflowId) seal()
597 0 : if (admitted.length === 0) return
598 0 : if (open) for (const sized of admitted) open.ops.push(sized)
599 0 : else open = { workflowId, ops: admitted }
600 : }
601 :
602 : // Admissions in mint order. Measurement runs user-controlled toJSON and
603 : // commitment runs settlement listeners; either may re-enter admit(). A
604 : // re-entrant admission takes the slot after the one in progress and is
605 : // committed by the outermost call, so the open group and the queue receive
606 : // ops in the order their base_versions were minted.
607 1 : const admitting: Admission[] = []
608 1 : let flushDeferred = false
609 :
610 0 : function admit(operations: GraphOperation[]): void {
611 0 : if (operations.length === 0) return
612 0 : const workflowId = deps.workflowId()
613 0 : const { minted, exhausted } = mintAdmission(operations, workflowId)
614 0 : const admission: Admission = {
615 : workflowId,
616 : minted,
617 : admitted: [],
618 : rejected: [],
619 : cause: undefined,
620 : exhausted
621 : }
622 0 : const nested = admitting.length > 0
623 0 : admitting.push(admission)
624 0 : if (!exhausted) Object.assign(admission, measureMintedOps(minted))
625 0 : if (nested) return
626 0 : commitAdmissions()
627 : }
628 :
629 0 : function commitAdmissions(): void {
630 0 : try {
631 0 : for (const admission of admitting) commitAdmission(admission)
632 : } finally {
633 0 : admitting.length = 0
634 : }
635 0 : if (!flushDeferred) return
636 0 : flushDeferred = false
637 0 : flush()
638 : }
639 :
640 0 : function commitAdmission({
641 : workflowId,
642 : minted,
643 : admitted,
644 : rejected,
645 : cause,
646 : exhausted
647 : }: Admission): void {
648 0 : if (detached) {
649 0 : settleUndeliverableInBoundedGroups(
650 : minted,
651 : notifyDetachSettlement,
652 : workflowId
653 : )
654 0 : return
655 : }
656 0 : const notifyFailure = guardedSettlementNotifier(
657 : 'failure_settling_agent_op_sender',
658 : workflowId === null ? 'unbound' : 'serialization_failed'
659 : )
660 0 : if (workflowId === null) {
661 0 : settleUndeliverableInBoundedGroups(minted, notifyFailure, workflowId)
662 0 : return
663 : }
664 0 : if (exhausted) {
665 0 : if (mayReportCounterExhaustion()) {
666 0 : reportDegraded(
667 : new RangeError(
668 : 'Agent human op counters left the safe integers; the edit was not sent'
669 : ),
670 : 'failure_minting_agent_op_counter'
671 : )
672 : }
673 0 : settleUndeliverableInBoundedGroups(
674 : minted,
675 : guardedSettlementNotifier(
676 : 'failure_settling_agent_op_sender',
677 : 'counter_exhausted'
678 : ),
679 : workflowId
680 : )
681 0 : return
682 : }
683 0 : appendToOpenGroup(workflowId, admitted)
684 0 : if (rejected.length === 0) return
685 0 : if (mayReportSerializationFailure()) {
686 0 : reportDegraded(cause, 'failure_serializing_agent_op_sender')
687 : }
688 0 : settleUndeliverableInBoundedGroups(rejected, notifyFailure, workflowId)
689 : }
690 :
691 1 : function seal(): void {
692 1 : if (!open) return
693 0 : const { workflowId, ops } = open
694 0 : open = null
695 0 : for (const chunk of chunkWireOps(ops)) {
696 0 : queue.push(toWireBatch(workflowId, chunk))
697 : }
698 : }
699 :
700 1 : function flush(): void {
701 1 : if (admitting.length > 0) {
702 0 : flushDeferred = true
703 0 : return
704 : }
705 1 : seal()
706 1 : pump()
707 : }
708 :
709 0 : function resultBatch(result: OpsResultView): InFlight | null {
710 0 : if (inFlight === null) return null
711 0 : if (result.workflowId && result.workflowId !== inFlight.workflowId)
712 0 : return null
713 0 : return inFlight
714 : }
715 :
716 0 : function resultOpIds(result: OpsResultView): string[] {
717 0 : return [
718 : ...result.applied,
719 : ...result.skipped,
720 : ...(result.failed?.op_id ? [result.failed.op_id] : [])
721 : ]
722 : }
723 :
724 0 : function settleResult(batch: InFlight, result: OpsResultView): void {
725 0 : retire(batch, 1)
726 0 : settle(
727 : {
728 : state: 'acknowledged',
729 : ops: batch.ops,
730 : result,
731 : ...(batch.sends > 1 && { resent: true })
732 : },
733 : 'host_result'
734 : )
735 : }
736 :
737 0 : function settleIdentifiedResult(
738 : batch: InFlight | null,
739 : result: OpsResultView,
740 : opIds: readonly string[]
741 : ): void {
742 0 : if (batch && opIds.some((opId) => batch.opIds.has(opId))) {
743 0 : settleResult(batch, result)
744 0 : return
745 : }
746 0 : if (opIds.some((opId) => retiredOpIds.has(opId))) drainStaleCredit()
747 : }
748 :
749 0 : function handleOpsResult(result: OpsResultView): void {
750 0 : if (detached) return
751 0 : if (inFlight === null && staleAnonymousBudget === 0) return
752 0 : const opIds = resultOpIds(result)
753 0 : const batch = resultBatch(result)
754 0 : if (opIds.length > 0) return settleIdentifiedResult(batch, result, opIds)
755 0 : if (batch === null || staleAnonymousBudget > 0) {
756 0 : drainStaleCredit()
757 0 : return
758 : }
759 0 : if (batch.sends === 0) return
760 0 : settleResult(batch, result)
761 : }
762 :
763 1 : const unsubscribe = deps.onOpsResult(handleOpsResult)
764 :
765 1 : return {
766 : enqueue(operations) {
767 0 : seal()
768 0 : admit(operations)
769 0 : flush()
770 : },
771 : admit,
772 : flush,
773 : pending() {
774 0 : return queue.length - queueHead + (inFlight ? 1 : 0) + (open ? 1 : 0)
775 : },
776 : pendingOps() {
777 0 : const batches = inFlight
778 : ? [{ workflowId: inFlight.workflowId, ops: inFlight.ops }]
779 : : []
780 0 : const unsealed = open
781 : ? [{ workflowId: open.workflowId, ops: open.ops.map((s) => s.op) }]
782 : : []
783 0 : const queued = queue
784 : .slice(queueHead)
785 : .map(({ workflowId, ops }) => ({ workflowId, ops }))
786 0 : return [...batches, ...queued, ...unsealed]
787 : },
788 : suspend() {
789 0 : suspended = true
790 : },
791 : resume() {
792 1 : suspended = false
793 1 : if (inFlight?.parked) {
794 0 : inFlight.parked = false
795 0 : transmit(inFlight, 0)
796 : }
797 : },
798 : retryTransport() {
799 0 : if (!inFlight?.awaitingTransport) return
800 0 : inFlight.awaitingTransport = false
801 0 : transmit(inFlight, 0)
802 : },
803 : abortIfUnbound(reason) {
804 1 : if (inFlight && deps.workflowId() !== inFlight.workflowId) {
805 0 : const previousAbortReason = pendingAbortReason
806 0 : pendingAbortReason = { workflowId: inFlight.workflowId, reason }
807 0 : try {
808 0 : settleUnbound(inFlight, 'unbound', reason)
809 : } finally {
810 0 : pendingAbortReason = previousAbortReason
811 : }
812 : }
813 : },
814 : abortAll() {
815 0 : const resetWorkflowId = deps.workflowId()
816 0 : if (resetWorkflowId !== null) {
817 0 : lastMintedVersions.delete(resetWorkflowId)
818 0 : lastSentVersions.delete(resetWorkflowId)
819 : }
820 0 : drainOutstanding(
821 : guardedSettlementNotifier(
822 : 'failure_settling_agent_op_sender_abort',
823 : 'doc_reset'
824 : )
825 : )
826 : },
827 : detach() {
828 0 : if (detached) return
829 0 : detached = true
830 0 : try {
831 0 : drainOutstanding(notifyDetachSettlement)
832 : // The listener is inert once detached. Release late-result bookkeeping
833 : // even if an injected unsubscribe implementation fails below.
834 0 : staleAnonymousBudget = 0
835 0 : retiredOpIds.clear()
836 : } finally {
837 0 : try {
838 0 : unsubscribe()
839 : } catch (cause) {
840 0 : reportDegraded(cause, 'failure_unsubscribing_agent_op_sender')
841 : }
842 : }
843 : }
844 : }
845 : }
|