Line data Source code
1 : import type {
2 : AxiosAdapter,
3 : AxiosInstance,
4 : AxiosResponse,
5 : InternalAxiosRequestConfig
6 : } from 'axios'
7 : import axios, { AxiosError } from 'axios'
8 :
9 : import { combineAbortSignals } from '@comfyorg/api-client/abortSignal'
10 : import type { RetryPolicy } from '@comfyorg/api-client/retryPolicy'
11 : import {
12 : canRetrySignInRefusal,
13 : nextRetry,
14 : signInRefusalDelayMs,
15 : sleep as sleepFor
16 : } from '@comfyorg/api-client/retryPolicy'
17 : import type { StatusRetryEnd } from '@comfyorg/api-client/statusRetryTelemetry'
18 : import {
19 : abortEnd,
20 : endForStop,
21 : trackStatusRetry
22 : } from '@comfyorg/api-client/statusRetryTelemetry'
23 :
24 : export interface AxiosRetryableStatusOptions {
25 : readonly policy?: RetryPolicy
26 : readonly sleep?: (ms: number, signal?: AbortSignal) => Promise<void>
27 : readonly random?: () => number
28 : }
29 :
30 0 : function retryAfterHeader(response: AxiosResponse): string | undefined {
31 0 : const value = response.headers['retry-after']
32 0 : return typeof value === 'string' || typeof value === 'number'
33 : ? String(value)
34 : : undefined
35 : }
36 :
37 110 : function timeBudget(timeoutMs: number | undefined) {
38 110 : const startedAt = Date.now()
39 110 : return () => (timeoutMs ? timeoutMs - (Date.now() - startedAt) : undefined)
40 : }
41 :
42 0 : function cancellationSignal(
43 : config: InternalAxiosRequestConfig
44 : ): AbortSignal | undefined {
45 0 : const signal =
46 : config.signal instanceof AbortSignal ? config.signal : undefined
47 0 : if (!config.cancelToken) return signal
48 :
49 0 : const controller = new AbortController()
50 0 : void config.cancelToken.promise.then(() => controller.abort())
51 0 : return signal
52 : ? combineAbortSignals([signal, controller.signal])
53 : : controller.signal
54 : }
55 :
56 0 : function fitsBudget(delay: number, remaining: number | undefined): boolean {
57 0 : return remaining === undefined || delay < remaining
58 : }
59 :
60 110 : async function runAttempt(
61 : adapter: AxiosAdapter,
62 : config: InternalAxiosRequestConfig,
63 : remaining: number | undefined
64 : ): Promise<AxiosResponse> {
65 110 : try {
66 110 : const response = await adapter(
67 : remaining === undefined
68 : ? config
69 : : { ...config, timeout: Math.max(1, remaining) }
70 : )
71 109 : response.config = config
72 109 : return response
73 : } catch (error) {
74 0 : if (axios.isAxiosError(error) && error.config) error.config = config
75 0 : throw error
76 : }
77 : }
78 :
79 0 : function isTimeout(error: unknown): boolean {
80 0 : if (!axios.isAxiosError(error)) return false
81 0 : if (error.code === AxiosError.ETIMEDOUT) return true
82 0 : return (
83 : error.code === AxiosError.ECONNABORTED && /timeout/i.test(error.message)
84 : )
85 : }
86 :
87 0 : function noResponseEnd(error: unknown): StatusRetryEnd {
88 0 : if (isTimeout(error)) return 'timeout'
89 : const cancelled =
90 : axios.isCancel(error) ||
91 : (axios.isAxiosError(error) &&
92 : (error.code === AxiosError.ERR_CANCELED ||
93 : error.code === AxiosError.ECONNABORTED))
94 0 : return cancelled ? 'cancelled' : 'network'
95 : }
96 :
97 : interface StatusRetryTracker {
98 : scheduled(status: number): void
99 : settle(end: StatusRetryEnd, attempts: number, status?: number): void
100 : }
101 :
102 110 : function statusRetryTracker(
103 : config: InternalAxiosRequestConfig
104 : ): StatusRetryTracker {
105 110 : let lastRetryStatus: number | undefined
106 110 : return {
107 : scheduled(status) {
108 0 : lastRetryStatus = status
109 : },
110 : settle(end, attempts, status) {
111 109 : trackStatusRetry({
112 : transport: 'axios',
113 : method: config.method ?? 'GET',
114 : url: () => axios.getUri(config),
115 : end,
116 : attempts,
117 : finalStatus: status ?? lastRetryStatus,
118 : retryScheduled: lastRetryStatus !== undefined
119 : })
120 : }
121 : }
122 : }
123 :
124 0 : function delayOrRethrow(
125 : error: unknown,
126 : config: InternalAxiosRequestConfig,
127 : attempt: number,
128 : remaining: number | undefined,
129 : tracker: StatusRetryTracker,
130 : { policy, random }: AxiosRetryableStatusOptions
131 : ): number {
132 0 : const failed = axios.isAxiosError(error) ? error.response : undefined
133 0 : if (!failed) {
134 0 : tracker.settle(noResponseEnd(error), attempt)
135 0 : throw error
136 : }
137 0 : const decision = nextRetry({
138 : method: config.method ?? 'GET',
139 : status: failed.status,
140 : readRetryAfter: () => retryAfterHeader(failed),
141 : attempt,
142 : policy,
143 : random
144 : })
145 0 : if (!decision.retry) {
146 0 : tracker.settle(endForStop(decision.reason), attempt, failed.status)
147 0 : throw error
148 : }
149 0 : tracker.scheduled(failed.status)
150 0 : if (!fitsBudget(decision.delayMs, remaining)) {
151 0 : tracker.settle('timeout', attempt)
152 0 : throw error
153 : }
154 0 : return decision.delayMs
155 : }
156 :
157 : type Pause = (
158 : delay: number,
159 : config: InternalAxiosRequestConfig,
160 : settleAbort: (signal: AbortSignal | undefined) => void
161 : ) => Promise<void>
162 :
163 110 : function resendsSignInRefusal(
164 : config: InternalAxiosRequestConfig,
165 : policy: RetryPolicy | undefined
166 : ): boolean {
167 110 : return (
168 : canRetrySignInRefusal(config.method ?? 'GET', policy) &&
169 : !(config.data instanceof ReadableStream)
170 : )
171 : }
172 :
173 0 : async function refusedWriteDelay(
174 : error: unknown,
175 : config: InternalAxiosRequestConfig,
176 : remaining: number | undefined,
177 : { policy, random }: AxiosRetryableStatusOptions
178 : ): Promise<number | undefined> {
179 0 : const failed = axios.isAxiosError(error) ? error.response : undefined
180 0 : if (!failed) return undefined
181 0 : const delay = await signInRefusalDelayMs({
182 : method: config.method ?? 'GET',
183 : status: failed.status,
184 : readRetryAfter: () => retryAfterHeader(failed),
185 : readBody: () => failed.data,
186 : policy,
187 : random
188 : })
189 0 : return delay !== undefined && fitsBudget(delay, remaining) ? delay : undefined
190 : }
191 :
192 0 : async function sendWriteAfterRefusal(
193 : adapter: AxiosAdapter,
194 : config: InternalAxiosRequestConfig,
195 : options: AxiosRetryableStatusOptions,
196 : pause: Pause
197 : ): Promise<AxiosResponse> {
198 0 : const remainingMs = timeBudget(config.timeout)
199 0 : const tracker = statusRetryTracker(config)
200 0 : try {
201 0 : return await runAttempt(adapter, config, remainingMs())
202 : } catch (error) {
203 0 : const delay = await refusedWriteDelay(error, config, remainingMs(), options)
204 0 : if (delay === undefined) throw error
205 0 : tracker.scheduled(503)
206 0 : await pause(delay, config, (signal) => tracker.settle(abortEnd(signal), 1))
207 0 : if (!fitsBudget(0, remainingMs())) {
208 0 : tracker.settle('timeout', 1)
209 0 : throw error
210 : }
211 : }
212 0 : try {
213 0 : const response = await runAttempt(adapter, config, remainingMs())
214 0 : tracker.settle('answered', 2, response.status)
215 0 : return response
216 : } catch (error) {
217 0 : const failed = axios.isAxiosError(error) ? error.response : undefined
218 0 : tracker.settle(
219 : failed ? 'answered' : noResponseEnd(error),
220 : 2,
221 : failed?.status
222 : )
223 0 : throw error
224 : }
225 : }
226 :
227 128 : function retryingAdapter(
228 : adapter: AxiosAdapter,
229 : options: AxiosRetryableStatusOptions
230 : ): AxiosAdapter {
231 128 : const { sleep = sleepFor } = options
232 :
233 0 : async function pause(
234 : delay: number,
235 : config: InternalAxiosRequestConfig,
236 : settleAbort: (signal: AbortSignal | undefined) => void
237 : ) {
238 0 : const signal = cancellationSignal(config)
239 0 : try {
240 0 : await sleep(delay, signal)
241 : } catch (error) {
242 0 : settleAbort(signal)
243 0 : if (!signal?.aborted) throw error
244 0 : throw new axios.CanceledError('canceled', config)
245 : }
246 : }
247 :
248 128 : return async (config) => {
249 128 : if (config.__skipStatusRetry) return adapter(config)
250 128 : if (resendsSignInRefusal(config, options.policy)) {
251 0 : return sendWriteAfterRefusal(adapter, config, options, pause)
252 : }
253 110 : const remainingMs = timeBudget(config.timeout)
254 110 : const tracker = statusRetryTracker(config)
255 110 : let lastError: unknown
256 :
257 110 : for (let attempt = 1; ; attempt++) {
258 110 : const remaining = remainingMs()
259 110 : if (attempt > 1 && remaining !== undefined && remaining <= 0) {
260 0 : tracker.settle('timeout', attempt - 1)
261 0 : throw lastError
262 : }
263 110 : try {
264 110 : const response = await runAttempt(adapter, config, remaining)
265 109 : tracker.settle('answered', attempt, response.status)
266 109 : return response
267 : } catch (error) {
268 0 : lastError = error
269 : const delay = delayOrRethrow(
270 : error,
271 : config,
272 : attempt,
273 : remainingMs(),
274 : tracker,
275 : options
276 : )
277 0 : await pause(delay, config, (signal) =>
278 : tracker.settle(abortEnd(signal), attempt)
279 : )
280 : }
281 : }
282 : }
283 : }
284 :
285 2356 : const retrying = new WeakSet<AxiosAdapter>()
286 :
287 : /**
288 : * Retries retryable statuses around each request's adapter, within its
289 : * `timeout`; a policy may opt in a write the sign-in check refused.
290 : */
291 2355 : export function attachRetryableStatusRetry(
292 : client: AxiosInstance,
293 : options: AxiosRetryableStatusOptions = {}
294 : ): void {
295 2355 : client.interceptors.request.use((config) => {
296 128 : const selected = config.adapter ?? client.defaults.adapter
297 128 : if (typeof selected === 'function' && retrying.has(selected)) return config
298 :
299 128 : const adapter = retryingAdapter(axios.getAdapter(selected), options)
300 128 : retrying.add(adapter)
301 128 : config.adapter = adapter
302 128 : return config
303 : })
304 : }
|