Line data Source code
1 : import { useTelemetry } from '@comfyorg/telemetry'
2 : import type {
3 : FetchStatusRetryMetadata,
4 : FetchStatusRetryOutcome
5 : } from '@comfyorg/telemetry/types'
6 :
7 : import { getFetchRouteTemplate } from './fetchRouteTemplate'
8 : import type { RetryStopReason } from './retryPolicy'
9 :
10 : /** How a request's retry loop ended. */
11 : export type StatusRetryEnd =
12 : | 'answered'
13 : | Exclude<RetryStopReason, 'not_retryable'>
14 : | 'timeout'
15 : | 'network'
16 : | 'cancelled'
17 :
18 : export interface StatusRetrySettlement {
19 : readonly end: StatusRetryEnd
20 : /** Requests sent, including the first. */
21 : readonly attempts: number
22 : readonly finalStatus?: number
23 : /** Whether an earlier answer was retryable and a retry was scheduled. */
24 : readonly retryScheduled: boolean
25 : }
26 :
27 0 : function isSuccessfulStatus(status: number | undefined): boolean {
28 0 : return status !== undefined && status >= 200 && status < 300
29 : }
30 :
31 : /** The outcome to record for a settled request, or `undefined` for none. */
32 109 : export function classifyStatusRetry({
33 : end,
34 : attempts,
35 : finalStatus,
36 : retryScheduled
37 : }: StatusRetrySettlement): FetchStatusRetryOutcome | undefined {
38 109 : switch (end) {
39 : case 'cancelled':
40 0 : return undefined
41 : case 'retry_after_too_long':
42 0 : return end
43 : case 'attempts_spent':
44 0 : return attempts > 1 ? 'exhausted' : undefined
45 : case 'timeout':
46 0 : return retryScheduled ? 'timeout_budget' : undefined
47 : case 'network':
48 0 : return retryScheduled ? 'failed' : undefined
49 : case 'answered':
50 109 : if (!retryScheduled) return undefined
51 109 : return isSuccessfulStatus(finalStatus) ? 'recovered' : 'failed'
52 : }
53 : }
54 :
55 : /** How the loop ended when it stopped on an answer. */
56 0 : export function endForStop(reason: RetryStopReason): StatusRetryEnd {
57 0 : return reason === 'not_retryable' ? 'answered' : reason
58 : }
59 :
60 2356 : const MAX_EVENTS_PER_KEY_PER_WINDOW = 20
61 2356 : const BUDGET_WINDOW_MS = 60_000
62 2356 : const budgetPerKey = new Map<string, { windowStart: number; sent: number }>()
63 :
64 : /** Forgets how many events each key has sent. */
65 : export function clearStatusRetryBudget(): void {
66 : budgetPerKey.clear()
67 : }
68 :
69 0 : function spendBudget(key: string): boolean {
70 0 : const now = Date.now()
71 0 : const budget = budgetPerKey.get(key)
72 0 : if (!budget || now - budget.windowStart >= BUDGET_WINDOW_MS) {
73 0 : budgetPerKey.set(key, { windowStart: now, sent: 1 })
74 0 : return true
75 : }
76 0 : if (budget.sent >= MAX_EVENTS_PER_KEY_PER_WINDOW) return false
77 0 : budget.sent += 1
78 0 : return true
79 : }
80 :
81 0 : function telemetryHost(host: string): string {
82 : const knownHost =
83 : host === location.host ||
84 : host === 'comfy.org' ||
85 : host.endsWith('.comfy.org')
86 0 : return knownHost ? host : 'other'
87 : }
88 :
89 0 : function describeUrl(
90 : url: string
91 : ): Pick<FetchStatusRetryMetadata, 'host' | 'route'> {
92 0 : const parsed = new URL(url, location.href)
93 0 : return {
94 : host: telemetryHost(parsed.host),
95 : route: getFetchRouteTemplate(parsed.pathname)
96 : }
97 : }
98 :
99 : interface StatusRetryReport extends StatusRetrySettlement {
100 : readonly transport: FetchStatusRetryMetadata['transport']
101 : readonly method: string
102 : readonly url: () => string
103 : }
104 :
105 109 : function recordStatusRetry(report: StatusRetryReport): void {
106 109 : const outcome = classifyStatusRetry(report)
107 109 : const telemetry = useTelemetry()
108 109 : if (!outcome || !telemetry) return
109 0 : const { host, route } = describeUrl(report.url())
110 0 : if (!spendBudget(`${host} ${route} ${outcome}`)) return
111 :
112 109 : telemetry.trackFetchStatusRetry({
113 : transport: report.transport,
114 : method: report.method.toUpperCase(),
115 : host,
116 : route,
117 : attempts: report.attempts,
118 : ...(report.finalStatus !== undefined && {
119 : final_status: report.finalStatus
120 : }),
121 : outcome
122 : })
123 : }
124 :
125 : /** Records one settled request that hit a retryable status; never throws. */
126 109 : export function trackStatusRetry(report: StatusRetryReport): void {
127 109 : try {
128 109 : recordStatusRetry(report)
129 : } catch (error) {
130 0 : console.warn('[telemetry] status retry event dropped', error)
131 : }
132 : }
133 :
134 : /** Whether a failed or aborted fetch was the request timeout, the caller, or the network. */
135 0 : export function abortEnd(signal: AbortSignal | undefined): StatusRetryEnd {
136 0 : if (!signal?.aborted) return 'network'
137 0 : const reason: unknown = signal.reason
138 0 : return reason instanceof DOMException && reason.name === 'TimeoutError'
139 : ? 'timeout'
140 : : 'cancelled'
141 : }
|