LCOV - code coverage report
Current view: top level - src/platform/telemetry/perf - bootstrapTracer.ts Coverage Total Hit
Test: ComfyUI E2E Coverage Lines: 79.5 % 83 66
Test Date: 2026-11-17 04:17:53

            Line data    Source code
       1              : /**
       2              :  * Structured performance tracing for the app startup/bootstrap sequence.
       3              :  *
       4              :  * Each phase maps to one of the known bottlenecks from profiling:
       5              :  *
       6              :  *   Phase                          | Typical range (cloud, Lighthouse)
       7              :  *   ───────────────────────────────┼──────────────────────────────────
       8              :  *   startup/entry-modules          | variable  (time origin → bootstrap(): HTML + static imports)
       9              :  *   startup/remote-config          | ~0–0.5 s  (pre-Firebase, blocking)
      10              :  *   startup/telemetry-init         | ~0–0.3 s
      11              :  *   startup/local-auth             | non-cloud (local web or desktop host session)
      12              :  *   startup/firebase-init          | ~0–0.1 s
      13              :  *   startup/router                 | ~0 s      (route table; cloud routes import)
      14              :  *   startup/sentry-init            | ~0–0.1 s
      15              :  *   auth-gate/initialized          | variable  (Firebase session restore + timeout/retry)
      16              :  *   auth-gate/user-store           | ~0–2.5 s  (GET /api/user/*)
      17              :  *   auth-gate/needs-login          | ~0 s      (computed resolution)
      18              :  *   bootstrap/settings             | ~0.2 s    (parallel, non-blocking)
      19              :  *   bootstrap/workflows            | ~0.3 s    (parallel, non-blocking)
      20              :  *   bootstrap/i18n                 | parallel  (custom node translations)
      21              :  *   bootstrap/graph-view-chunk     | variable  (lazy GraphView route import)
      22              :  *   bootstrap/new-user             | ~0 s      (new-user defaults)
      23              :  *   bootstrap/extensions-load      | ~0.5 s    (fetch + import /extensions JS)
      24              :  *     ├ extensions-load-core       | subset    (core entry-point import)
      25              :  *     └ extensions-load-custom     | subset    (backend-provided extension imports)
      26              :  *   bootstrap/deployment-pick      | cloud     (deployment lookup)
      27              :  *   bootstrap/sync-workflows       | ~0 s      (workflow store sync)
      28              :  *   bootstrap/extensions-init      | ~0.3 s    (extension.init() hooks)
      29              :  *   bootstrap/object-info          | ~0.7 s    (GET /api/object_info)
      30              :  *   bootstrap/extensions           | ~0.5 s    (addCustomNodeDefs + registerCustomNodes)
      31              :  *   bootstrap/vue-node-defs        | ~0.1 s    (node def store update)
      32              :  *   bootstrap/extensions-setup     | ~0.5 s    (extension.setup() hooks)
      33              :  *   bootstrap/workflow-restore     | ~0.3 s    (initial workflow, tabs, URL template)
      34              :  *
      35              :  * The two `extensions-load-*` rows are nested inside `extensions-load` rather
      36              :  * than replacing it: the aggregate stays intact for the existing monitors, and
      37              :  * the residual (aggregate minus both children) is `api.getExtensions()` plus
      38              :  * bookkeeping. So read the children and the residual against the aggregate —
      39              :  * never sum rows across the flattened phase map, which would double-count
      40              :  * extension load time.
      41              :  *
      42              :  * These children add two to `phase_count` when both run, so thresholds based
      43              :  * on the old count must shift. Each duration is rounded independently, so the
      44              :  * residual can be off by a millisecond or two in either direction and can read
      45              :  * negative on a fast load. The phase map carries only the rounded values.
      46              :  *
      47              :  * Startup ends when the loading screen comes down, not when any one subsystem
      48              :  * finishes. `complete()` is therefore called from the single place that owns
      49              :  * that transition (`GraphCanvas`'s onMounted `finally`), so the trace covers
      50              :  * the extension and object-info phases that run inside `ComfyApp.setup()` and
      51              :  * so a startup that threw still reports. A startup that never gets there is
      52              :  * reported by `armWatchdog()`, armed from `main.ts` before blocking startup
      53              :  * work begins.
      54              :  *
      55              :  * Usage:
      56              :  *   import { bootstrapTracer } from '@/platform/telemetry/perf/bootstrapTracer'
      57              :  *
      58              :  *   await bootstrapTracer.settle('bootstrap/object-info', () => this.getNodeDefs())
      59              :  */
      60              : import { useTelemetry } from '@comfyorg/telemetry'
      61              : import { getDatadogRum } from '@comfyorg/telemetry/rumContext'
      62              : import { TelemetryEvents } from '@comfyorg/telemetry/types'
      63              : import type { BootstrapCompleteMetadata } from '@comfyorg/telemetry/types'
      64              : 
      65              : import { perfMark, perfMeasureSinceOrigin, perfPoint } from './perfMark'
      66              : import type { PerfSpan } from './perfMark'
      67              : 
      68              : /**
      69              :  * Startup past this is abnormal by any reading — the slowest healthy cloud
      70              :  * load profiled is ~5 s, and the reports that prompted this instrumentation
      71              :  * were minutes.
      72              :  */
      73         2356 : const BOOTSTRAP_WATCHDOG_MS = 30_000
      74              : 
      75            0 : function toRumTimingName(name: string): string {
      76            0 :   return name.replace(/\//g, '.')
      77              : }
      78              : 
      79              : const BOOTSTRAP_PHASES = {
      80              :   startup: [
      81              :     'entry-modules',
      82              :     'remote-config',
      83              :     'telemetry-init',
      84              :     'local-auth',
      85              :     'firebase-init',
      86              :     'router',
      87              :     'sentry-init'
      88              :   ],
      89              :   'auth-gate': [
      90              :     'initialized',
      91              :     'identity',
      92              :     'sign-in',
      93              :     'user-store',
      94              :     'needs-login'
      95              :   ],
      96              :   bootstrap: [
      97              :     'settings',
      98              :     'workflows',
      99              :     'i18n',
     100              :     'graph-view-chunk',
     101              :     'new-user',
     102              :     'extensions-load',
     103              :     'extensions-load-core',
     104              :     'extensions-load-custom',
     105              :     'deployment-pick',
     106              :     'sync-workflows',
     107              :     'extensions-init',
     108              :     'object-info',
     109              :     'extensions',
     110              :     'vue-node-defs',
     111              :     'extensions-setup',
     112              :     'workflow-restore'
     113              :   ]
     114              : } as const satisfies Record<string, readonly string[]>
     115              : 
     116              : type BootstrapPhaseNamespace = keyof typeof BOOTSTRAP_PHASES
     117              : 
     118              : type BootstrapSubphase<Namespace extends BootstrapPhaseNamespace> =
     119              :   (typeof BOOTSTRAP_PHASES)[Namespace][number]
     120              : 
     121              : export type BootstrapPhase = {
     122              :   [
     123              :     Namespace in BootstrapPhaseNamespace
     124              :   ]: `${Namespace}/${BootstrapSubphase<Namespace>}`
     125              : }[BootstrapPhaseNamespace]
     126              : 
     127         2355 : function untracedMs(rows: BootstrapPhaseTiming[], totalMs: number): number {
     128         2355 :   let tracedMs = 0
     129         2355 :   let coveredUntil = 0
     130         2355 :   for (const { startMs, durationMs } of rows) {
     131        54165 :     const endMs = Math.min(startMs + durationMs, totalMs)
     132        54165 :     tracedMs += Math.max(0, endMs - Math.max(startMs, coveredUntil))
     133        54165 :     coveredUntil = Math.max(coveredUntil, endMs)
     134              :   }
     135         2355 :   return totalMs - tracedMs
     136              : }
     137              : 
     138         2356 : class HiddenTimeMeter {
     139              :   private _hiddenMs = 0
     140              :   private _hiddenSince: number | undefined
     141              :   private _listening = new AbortController()
     142              : 
     143              :   constructor() {
     144         2356 :     if (typeof document === 'undefined') return
     145         2356 :     const buffered = performance.getEntriesByType('visibility-state')
     146         2356 :     if (buffered.length > 0) {
     147         2356 :       for (const entry of buffered)
     148         2356 :         this._record(entry.name === 'hidden', entry.startTime)
     149            0 :     } else if (document.visibilityState === 'hidden') {
     150            0 :       this._hiddenSince = 0
     151              :     }
     152         2356 :     document.addEventListener(
     153              :       'visibilitychange',
     154              :       () =>
     155              :         this._record(document.visibilityState === 'hidden', performance.now()),
     156              :       { signal: this._listening.signal }
     157              :     )
     158              :   }
     159              : 
     160              :   private _record(hidden: boolean, atMs: number): void {
     161         2356 :     if (hidden) {
     162            0 :       this._hiddenSince ??= atMs
     163         2356 :     } else if (this._hiddenSince !== undefined) {
     164            0 :       this._hiddenMs += atMs - this._hiddenSince
     165            0 :       this._hiddenSince = undefined
     166              :     }
     167              :   }
     168              : 
     169              :   stop(): void {
     170         2355 :     this._listening.abort()
     171              :   }
     172              : 
     173              :   hiddenMsUntil(nowMs: number): number {
     174         2355 :     const open = this._hiddenSince === undefined ? 0 : nowMs - this._hiddenSince
     175         2355 :     return Math.round(this._hiddenMs + Math.max(0, open))
     176              :   }
     177              : }
     178              : 
     179              : export interface BootstrapPhaseTiming {
     180              :   name: BootstrapPhase
     181              :   startMs: number
     182              :   durationMs: number
     183              : }
     184              : 
     185         2356 : export class BootstrapTracer {
     186              :   private _spans = new Map<BootstrapPhase, PerfSpan>()
     187              :   private _timings: BootstrapPhaseTiming[] = []
     188              :   private _completed = false
     189              :   private _outcome: 'completed' | 'failed' | undefined
     190              :   private _timeoutReported = false
     191              :   private _watchdog: ReturnType<typeof setTimeout> | undefined
     192              :   private _hidden = new HiddenTimeMeter()
     193              : 
     194              :   /**
     195              :    * Start timing a phase. Returns a handle; call `.stop()` when complete.
     196              :    * Calling startPhase for the same phase twice is safe (second call is a no-op
     197              :    * and returns the existing span's handle).
     198              :    */
     199              :   startPhase(phase: BootstrapPhase): { stop: () => number } {
     200        51810 :     if (this._spans.has(phase)) {
     201              :       // Already running — return a no-op handle so callers don't have to guard
     202            0 :       return { stop: () => 0 }
     203              :     }
     204        51810 :     this._spans.set(phase, perfMark(phase))
     205        51810 :     return { stop: () => this._stopPhase(phase) }
     206              :   }
     207              : 
     208              :   private _stopPhase(phase: BootstrapPhase): number {
     209        51810 :     const span = this._spans.get(phase)
     210        51810 :     if (!span) return 0
     211        51810 :     this._spans.delete(phase)
     212        51810 :     const durationMs = span.stop()
     213        51810 :     this._publishRumTiming(phase)
     214        51810 :     this._timings.push({
     215              :       name: phase,
     216              :       startMs: Math.round(span.startMs),
     217              :       durationMs: Math.round(durationMs)
     218              :     })
     219        51810 :     return durationMs
     220              :   }
     221              : 
     222              :   /**
     223              :    * Time `work` and record the phase whether it resolves or rejects, then
     224              :    * re-settle exactly as `work` did.
     225              :    *
     226              :    * A phase stopped only on the success path goes unrecorded precisely when
     227              :    * startup went wrong, so the surviving sessions are the fast ones and every
     228              :    * percentile reads better than reality.
     229              :    */
     230              :   async settle<T>(phase: BootstrapPhase, work: () => Promise<T>): Promise<T> {
     231        44745 :     const span = this.startPhase(phase)
     232        44745 :     try {
     233        44745 :       return await work()
     234              :     } finally {
     235        44745 :       span.stop()
     236              :     }
     237              :   }
     238              : 
     239              :   /**
     240              :    * Record a phase that began at the navigation time origin, such as the
     241              :    * static module graph evaluated before any startup code can run.
     242              :    */
     243              :   recordSinceTimeOrigin(phase: BootstrapPhase): void {
     244         2355 :     const durationMs = perfMeasureSinceOrigin(phase)
     245         2355 :     this._publishRumTiming(phase)
     246         2355 :     this._timings.push({
     247              :       name: phase,
     248              :       startMs: 0,
     249              :       durationMs: Math.round(durationMs)
     250              :     })
     251              :   }
     252              : 
     253              :   /**
     254              :    * Record a named milestone with no duration (e.g. "app shell mounted").
     255              :    */
     256              :   milestone(name: string): void {
     257         4710 :     const milestoneName = `bootstrap/milestone/${name}`
     258         4710 :     perfPoint(milestoneName)
     259         4710 :     this._publishRumTiming(milestoneName)
     260              :   }
     261              : 
     262              :   private _publishRumTiming(name: string): void {
     263        58875 :     getDatadogRum()?.addTiming(toRumTimingName(name))
     264              :   }
     265              : 
     266              :   /**
     267              :    * Completed phases, in the order they started.
     268              :    *
     269              :    * Built from what this tracer measured rather than re-scanned out of
     270              :    * `performance.getEntriesByType('measure')`: that buffer is shared with
     271              :    * every other measure in the app and can be cleared by anyone.
     272              :    */
     273              :   summary(): BootstrapPhaseTiming[] {
     274         2355 :     return [...this._timings].sort((a, b) => a.startMs - b.startMs)
     275              :   }
     276              : 
     277              :   /**
     278              :    * `timed_out` is not terminal: startup can still finish after the watchdog
     279              :    * fires, and the outcome then becomes `completed` or `failed`.
     280              :    */
     281              :   get outcome(): 'completed' | 'failed' | 'timed_out' | 'pending' {
     282            0 :     return this._outcome ?? (this._timeoutReported ? 'timed_out' : 'pending')
     283              :   }
     284              : 
     285              :   /**
     286              :    * Close out startup: stop any phase still open, emit one aggregate event
     287              :    * carrying total wall-clock and the per-phase breakdown, and log the trace
     288              :    * locally. Idempotent — only the first call reports.
     289              :    */
     290              :   complete(outcome: 'completed' | 'failed' = 'completed'): void {
     291         2355 :     if (this._completed) return
     292         2355 :     this._completed = true
     293         2355 :     this._outcome = outcome
     294         2355 :     clearTimeout(this._watchdog)
     295         2355 :     for (const phase of Array.from(this._spans.keys())) this._stopPhase(phase)
     296         2355 :     this._report(outcome)
     297         2355 :     this._hidden.stop()
     298              :   }
     299              : 
     300              :   /**
     301              :    * Report startup as still running if it has not completed by `deadlineMs`.
     302              :    *
     303              :    * A session that hangs never reaches `complete()`, so without this the loads
     304              :    * users actually report — the ones that spin for minutes — are the only ones
     305              :    * that produce no data at all. The watchdog row does not end the trace: the
     306              :    * terminal row still follows if startup eventually finishes.
     307              :    */
     308              :   armWatchdog(deadlineMs = BOOTSTRAP_WATCHDOG_MS): void {
     309         2355 :     clearTimeout(this._watchdog)
     310         2355 :     if (this._completed || this._timeoutReported) return
     311         2355 :     this._watchdog = setTimeout(() => {
     312            0 :       if (this._completed || this._timeoutReported) return
     313            0 :       this._timeoutReported = true
     314            0 :       this._report('timed_out', [...this._spans.keys()])
     315              :     }, deadlineMs)
     316              :   }
     317              : 
     318              :   suspendWatchdog(): void {
     319            0 :     clearTimeout(this._watchdog)
     320            0 :     this._watchdog = undefined
     321              :   }
     322              : 
     323              :   resumeWatchdog(): void {
     324         2355 :     if (!this._completed && this._watchdog === undefined) this.armWatchdog()
     325              :   }
     326              : 
     327              :   private _report(
     328              :     outcome: BootstrapCompleteMetadata['outcome'],
     329              :     pending?: BootstrapPhase[]
     330              :   ): void {
     331         2355 :     try {
     332         2355 :       const totalMs = Math.round(performance.now())
     333         2355 :       const rows = this.summary()
     334         2355 :       const metadata: BootstrapCompleteMetadata = {
     335              :         total_ms: totalMs,
     336              :         outcome,
     337              :         phase_count: rows.length,
     338              :         untraced_ms: untracedMs(rows, totalMs),
     339              :         hidden_ms: this._hidden.hiddenMsUntil(totalMs),
     340              :         phases: Object.fromEntries(rows.map((r) => [r.name, r.durationMs])),
     341              :         ...(pending?.length ? { pending } : {})
     342              :       }
     343         2355 :       getDatadogRum()?.addAction(TelemetryEvents.BOOTSTRAP_COMPLETE, metadata)
     344         2355 :       useTelemetry()?.trackBootstrapComplete(metadata)
     345         2355 :       if (outcome === 'completed') getDatadogRum()?.setViewLoadingTime()
     346         2355 :       this._logSummary(rows, totalMs)
     347              :     } catch {
     348            0 :       return
     349              :     }
     350              :   }
     351              : 
     352              :   /**
     353              :    * Log the startup summary to the console in a compact table.
     354              :    * Only emits in dev or when DEBUG_PERF is set.
     355              :    */
     356              :   private _logSummary(rows: BootstrapPhaseTiming[], totalMs: number): void {
     357         2355 :     if (!import.meta.env.DEV && localStorage.getItem('DEBUG_PERF') !== 'true')
     358         2355 :       return
     359            0 :     if (rows.length === 0) return
     360            0 :     console.warn(
     361              :       '[Bootstrap] Startup trace\n' +
     362              :         rows
     363              :           .map((r) => `  ${r.startMs}ms  ${r.name}: ${r.durationMs}ms`)
     364              :           .join('\n') +
     365              :         `\n  Total wall-clock: ${totalMs}ms` +
     366              :         `\n  Untraced: ${untracedMs(rows, totalMs)}ms`
     367              :     )
     368              :   }
     369              : }
     370              : 
     371              : /** Singleton tracer — import and use directly. */
     372         2356 : export const bootstrapTracer = new BootstrapTracer()
        

Generated by: LCOV version 2.0-1