LCOV - code coverage report
Current view: top level - src/utils - axiosRetryableStatus.ts Coverage Total Hit
Test: ComfyUI E2E Coverage Lines: 34.9 % 109 38
Test Date: 2026-11-17 04:17:53

            Line data    Source code
       1              : import type {
       2              :   AxiosAdapter,
       3              :   AxiosInstance,
       4              :   AxiosResponse,
       5              :   InternalAxiosRequestConfig
       6              : } from 'axios'
       7              : import axios, { AxiosError } from 'axios'
       8              : 
       9              : import { combineAbortSignals } from '@comfyorg/api-client/abortSignal'
      10              : import type { RetryPolicy } from '@comfyorg/api-client/retryPolicy'
      11              : import {
      12              :   canRetrySignInRefusal,
      13              :   nextRetry,
      14              :   signInRefusalDelayMs,
      15              :   sleep as sleepFor
      16              : } from '@comfyorg/api-client/retryPolicy'
      17              : import type { StatusRetryEnd } from '@comfyorg/api-client/statusRetryTelemetry'
      18              : import {
      19              :   abortEnd,
      20              :   endForStop,
      21              :   trackStatusRetry
      22              : } from '@comfyorg/api-client/statusRetryTelemetry'
      23              : 
      24              : export interface AxiosRetryableStatusOptions {
      25              :   readonly policy?: RetryPolicy
      26              :   readonly sleep?: (ms: number, signal?: AbortSignal) => Promise<void>
      27              :   readonly random?: () => number
      28              : }
      29              : 
      30            0 : function retryAfterHeader(response: AxiosResponse): string | undefined {
      31            0 :   const value = response.headers['retry-after']
      32            0 :   return typeof value === 'string' || typeof value === 'number'
      33              :     ? String(value)
      34              :     : undefined
      35              : }
      36              : 
      37          110 : function timeBudget(timeoutMs: number | undefined) {
      38          110 :   const startedAt = Date.now()
      39          110 :   return () => (timeoutMs ? timeoutMs - (Date.now() - startedAt) : undefined)
      40              : }
      41              : 
      42            0 : function cancellationSignal(
      43              :   config: InternalAxiosRequestConfig
      44              : ): AbortSignal | undefined {
      45            0 :   const signal =
      46              :     config.signal instanceof AbortSignal ? config.signal : undefined
      47            0 :   if (!config.cancelToken) return signal
      48              : 
      49            0 :   const controller = new AbortController()
      50            0 :   void config.cancelToken.promise.then(() => controller.abort())
      51            0 :   return signal
      52              :     ? combineAbortSignals([signal, controller.signal])
      53              :     : controller.signal
      54              : }
      55              : 
      56            0 : function fitsBudget(delay: number, remaining: number | undefined): boolean {
      57            0 :   return remaining === undefined || delay < remaining
      58              : }
      59              : 
      60          110 : async function runAttempt(
      61              :   adapter: AxiosAdapter,
      62              :   config: InternalAxiosRequestConfig,
      63              :   remaining: number | undefined
      64              : ): Promise<AxiosResponse> {
      65          110 :   try {
      66          110 :     const response = await adapter(
      67              :       remaining === undefined
      68              :         ? config
      69              :         : { ...config, timeout: Math.max(1, remaining) }
      70              :     )
      71          109 :     response.config = config
      72          109 :     return response
      73              :   } catch (error) {
      74            0 :     if (axios.isAxiosError(error) && error.config) error.config = config
      75            0 :     throw error
      76              :   }
      77              : }
      78              : 
      79            0 : function isTimeout(error: unknown): boolean {
      80            0 :   if (!axios.isAxiosError(error)) return false
      81            0 :   if (error.code === AxiosError.ETIMEDOUT) return true
      82            0 :   return (
      83              :     error.code === AxiosError.ECONNABORTED && /timeout/i.test(error.message)
      84              :   )
      85              : }
      86              : 
      87            0 : function noResponseEnd(error: unknown): StatusRetryEnd {
      88            0 :   if (isTimeout(error)) return 'timeout'
      89              :   const cancelled =
      90              :     axios.isCancel(error) ||
      91              :     (axios.isAxiosError(error) &&
      92              :       (error.code === AxiosError.ERR_CANCELED ||
      93              :         error.code === AxiosError.ECONNABORTED))
      94            0 :   return cancelled ? 'cancelled' : 'network'
      95              : }
      96              : 
      97              : interface StatusRetryTracker {
      98              :   scheduled(status: number): void
      99              :   settle(end: StatusRetryEnd, attempts: number, status?: number): void
     100              : }
     101              : 
     102          110 : function statusRetryTracker(
     103              :   config: InternalAxiosRequestConfig
     104              : ): StatusRetryTracker {
     105          110 :   let lastRetryStatus: number | undefined
     106          110 :   return {
     107              :     scheduled(status) {
     108            0 :       lastRetryStatus = status
     109              :     },
     110              :     settle(end, attempts, status) {
     111          109 :       trackStatusRetry({
     112              :         transport: 'axios',
     113              :         method: config.method ?? 'GET',
     114              :         url: () => axios.getUri(config),
     115              :         end,
     116              :         attempts,
     117              :         finalStatus: status ?? lastRetryStatus,
     118              :         retryScheduled: lastRetryStatus !== undefined
     119              :       })
     120              :     }
     121              :   }
     122              : }
     123              : 
     124            0 : function delayOrRethrow(
     125              :   error: unknown,
     126              :   config: InternalAxiosRequestConfig,
     127              :   attempt: number,
     128              :   remaining: number | undefined,
     129              :   tracker: StatusRetryTracker,
     130              :   { policy, random }: AxiosRetryableStatusOptions
     131              : ): number {
     132            0 :   const failed = axios.isAxiosError(error) ? error.response : undefined
     133            0 :   if (!failed) {
     134            0 :     tracker.settle(noResponseEnd(error), attempt)
     135            0 :     throw error
     136              :   }
     137            0 :   const decision = nextRetry({
     138              :     method: config.method ?? 'GET',
     139              :     status: failed.status,
     140              :     readRetryAfter: () => retryAfterHeader(failed),
     141              :     attempt,
     142              :     policy,
     143              :     random
     144              :   })
     145            0 :   if (!decision.retry) {
     146            0 :     tracker.settle(endForStop(decision.reason), attempt, failed.status)
     147            0 :     throw error
     148              :   }
     149            0 :   tracker.scheduled(failed.status)
     150            0 :   if (!fitsBudget(decision.delayMs, remaining)) {
     151            0 :     tracker.settle('timeout', attempt)
     152            0 :     throw error
     153              :   }
     154            0 :   return decision.delayMs
     155              : }
     156              : 
     157              : type Pause = (
     158              :   delay: number,
     159              :   config: InternalAxiosRequestConfig,
     160              :   settleAbort: (signal: AbortSignal | undefined) => void
     161              : ) => Promise<void>
     162              : 
     163          110 : function resendsSignInRefusal(
     164              :   config: InternalAxiosRequestConfig,
     165              :   policy: RetryPolicy | undefined
     166              : ): boolean {
     167          110 :   return (
     168              :     canRetrySignInRefusal(config.method ?? 'GET', policy) &&
     169              :     !(config.data instanceof ReadableStream)
     170              :   )
     171              : }
     172              : 
     173            0 : async function refusedWriteDelay(
     174              :   error: unknown,
     175              :   config: InternalAxiosRequestConfig,
     176              :   remaining: number | undefined,
     177              :   { policy, random }: AxiosRetryableStatusOptions
     178              : ): Promise<number | undefined> {
     179            0 :   const failed = axios.isAxiosError(error) ? error.response : undefined
     180            0 :   if (!failed) return undefined
     181            0 :   const delay = await signInRefusalDelayMs({
     182              :     method: config.method ?? 'GET',
     183              :     status: failed.status,
     184              :     readRetryAfter: () => retryAfterHeader(failed),
     185              :     readBody: () => failed.data,
     186              :     policy,
     187              :     random
     188              :   })
     189            0 :   return delay !== undefined && fitsBudget(delay, remaining) ? delay : undefined
     190              : }
     191              : 
     192            0 : async function sendWriteAfterRefusal(
     193              :   adapter: AxiosAdapter,
     194              :   config: InternalAxiosRequestConfig,
     195              :   options: AxiosRetryableStatusOptions,
     196              :   pause: Pause
     197              : ): Promise<AxiosResponse> {
     198            0 :   const remainingMs = timeBudget(config.timeout)
     199            0 :   const tracker = statusRetryTracker(config)
     200            0 :   try {
     201            0 :     return await runAttempt(adapter, config, remainingMs())
     202              :   } catch (error) {
     203            0 :     const delay = await refusedWriteDelay(error, config, remainingMs(), options)
     204            0 :     if (delay === undefined) throw error
     205            0 :     tracker.scheduled(503)
     206            0 :     await pause(delay, config, (signal) => tracker.settle(abortEnd(signal), 1))
     207            0 :     if (!fitsBudget(0, remainingMs())) {
     208            0 :       tracker.settle('timeout', 1)
     209            0 :       throw error
     210              :     }
     211              :   }
     212            0 :   try {
     213            0 :     const response = await runAttempt(adapter, config, remainingMs())
     214            0 :     tracker.settle('answered', 2, response.status)
     215            0 :     return response
     216              :   } catch (error) {
     217            0 :     const failed = axios.isAxiosError(error) ? error.response : undefined
     218            0 :     tracker.settle(
     219              :       failed ? 'answered' : noResponseEnd(error),
     220              :       2,
     221              :       failed?.status
     222              :     )
     223            0 :     throw error
     224              :   }
     225              : }
     226              : 
     227          128 : function retryingAdapter(
     228              :   adapter: AxiosAdapter,
     229              :   options: AxiosRetryableStatusOptions
     230              : ): AxiosAdapter {
     231          128 :   const { sleep = sleepFor } = options
     232              : 
     233            0 :   async function pause(
     234              :     delay: number,
     235              :     config: InternalAxiosRequestConfig,
     236              :     settleAbort: (signal: AbortSignal | undefined) => void
     237              :   ) {
     238            0 :     const signal = cancellationSignal(config)
     239            0 :     try {
     240            0 :       await sleep(delay, signal)
     241              :     } catch (error) {
     242            0 :       settleAbort(signal)
     243            0 :       if (!signal?.aborted) throw error
     244            0 :       throw new axios.CanceledError('canceled', config)
     245              :     }
     246              :   }
     247              : 
     248          128 :   return async (config) => {
     249          128 :     if (config.__skipStatusRetry) return adapter(config)
     250          128 :     if (resendsSignInRefusal(config, options.policy)) {
     251            0 :       return sendWriteAfterRefusal(adapter, config, options, pause)
     252              :     }
     253          110 :     const remainingMs = timeBudget(config.timeout)
     254          110 :     const tracker = statusRetryTracker(config)
     255          110 :     let lastError: unknown
     256              : 
     257          110 :     for (let attempt = 1; ; attempt++) {
     258          110 :       const remaining = remainingMs()
     259          110 :       if (attempt > 1 && remaining !== undefined && remaining <= 0) {
     260            0 :         tracker.settle('timeout', attempt - 1)
     261            0 :         throw lastError
     262              :       }
     263          110 :       try {
     264          110 :         const response = await runAttempt(adapter, config, remaining)
     265          109 :         tracker.settle('answered', attempt, response.status)
     266          109 :         return response
     267              :       } catch (error) {
     268            0 :         lastError = error
     269              :         const delay = delayOrRethrow(
     270              :           error,
     271              :           config,
     272              :           attempt,
     273              :           remainingMs(),
     274              :           tracker,
     275              :           options
     276              :         )
     277            0 :         await pause(delay, config, (signal) =>
     278              :           tracker.settle(abortEnd(signal), attempt)
     279              :         )
     280              :       }
     281              :     }
     282              :   }
     283              : }
     284              : 
     285         2356 : const retrying = new WeakSet<AxiosAdapter>()
     286              : 
     287              : /**
     288              :  * Retries retryable statuses around each request's adapter, within its
     289              :  * `timeout`; a policy may opt in a write the sign-in check refused.
     290              :  */
     291         2355 : export function attachRetryableStatusRetry(
     292              :   client: AxiosInstance,
     293              :   options: AxiosRetryableStatusOptions = {}
     294              : ): void {
     295         2355 :   client.interceptors.request.use((config) => {
     296          128 :     const selected = config.adapter ?? client.defaults.adapter
     297          128 :     if (typeof selected === 'function' && retrying.has(selected)) return config
     298              : 
     299          128 :     const adapter = retryingAdapter(axios.getAdapter(selected), options)
     300          128 :     retrying.add(adapter)
     301          128 :     config.adapter = adapter
     302          128 :     return config
     303              :   })
     304              : }
        

Generated by: LCOV version 2.0-1