Line data Source code
1 : /**
2 : * Structured performance tracing for the app startup/bootstrap sequence.
3 : *
4 : * Each phase maps to one of the known bottlenecks from profiling:
5 : *
6 : * Phase | Typical range (cloud, Lighthouse)
7 : * ───────────────────────────────┼──────────────────────────────────
8 : * startup/entry-modules | variable (time origin → bootstrap(): HTML + static imports)
9 : * startup/remote-config | ~0–0.5 s (pre-Firebase, blocking)
10 : * startup/telemetry-init | ~0–0.3 s
11 : * startup/local-auth | non-cloud (local web or desktop host session)
12 : * startup/firebase-init | ~0–0.1 s
13 : * startup/router | ~0 s (route table; cloud routes import)
14 : * startup/sentry-init | ~0–0.1 s
15 : * auth-gate/initialized | variable (Firebase session restore + timeout/retry)
16 : * auth-gate/user-store | ~0–2.5 s (GET /api/user/*)
17 : * auth-gate/needs-login | ~0 s (computed resolution)
18 : * bootstrap/settings | ~0.2 s (parallel, non-blocking)
19 : * bootstrap/workflows | ~0.3 s (parallel, non-blocking)
20 : * bootstrap/i18n | parallel (custom node translations)
21 : * bootstrap/graph-view-chunk | variable (lazy GraphView route import)
22 : * bootstrap/new-user | ~0 s (new-user defaults)
23 : * bootstrap/extensions-load | ~0.5 s (fetch + import /extensions JS)
24 : * ├ extensions-load-core | subset (core entry-point import)
25 : * └ extensions-load-custom | subset (backend-provided extension imports)
26 : * bootstrap/deployment-pick | cloud (deployment lookup)
27 : * bootstrap/sync-workflows | ~0 s (workflow store sync)
28 : * bootstrap/extensions-init | ~0.3 s (extension.init() hooks)
29 : * bootstrap/object-info | ~0.7 s (GET /api/object_info)
30 : * bootstrap/extensions | ~0.5 s (addCustomNodeDefs + registerCustomNodes)
31 : * bootstrap/vue-node-defs | ~0.1 s (node def store update)
32 : * bootstrap/extensions-setup | ~0.5 s (extension.setup() hooks)
33 : * bootstrap/workflow-restore | ~0.3 s (initial workflow, tabs, URL template)
34 : *
35 : * The two `extensions-load-*` rows are nested inside `extensions-load` rather
36 : * than replacing it: the aggregate stays intact for the existing monitors, and
37 : * the residual (aggregate minus both children) is `api.getExtensions()` plus
38 : * bookkeeping. So read the children and the residual against the aggregate —
39 : * never sum rows across the flattened phase map, which would double-count
40 : * extension load time.
41 : *
42 : * These children add two to `phase_count` when both run, so thresholds based
43 : * on the old count must shift. Each duration is rounded independently, so the
44 : * residual can be off by a millisecond or two in either direction and can read
45 : * negative on a fast load. The phase map carries only the rounded values.
46 : *
47 : * Startup ends when the loading screen comes down, not when any one subsystem
48 : * finishes. `complete()` is therefore called from the single place that owns
49 : * that transition (`GraphCanvas`'s onMounted `finally`), so the trace covers
50 : * the extension and object-info phases that run inside `ComfyApp.setup()` and
51 : * so a startup that threw still reports. A startup that never gets there is
52 : * reported by `armWatchdog()`, armed from `main.ts` before blocking startup
53 : * work begins.
54 : *
55 : * Usage:
56 : * import { bootstrapTracer } from '@/platform/telemetry/perf/bootstrapTracer'
57 : *
58 : * await bootstrapTracer.settle('bootstrap/object-info', () => this.getNodeDefs())
59 : */
60 : import { useTelemetry } from '@comfyorg/telemetry'
61 : import { getDatadogRum } from '@comfyorg/telemetry/rumContext'
62 : import { TelemetryEvents } from '@comfyorg/telemetry/types'
63 : import type { BootstrapCompleteMetadata } from '@comfyorg/telemetry/types'
64 :
65 : import { perfMark, perfMeasureSinceOrigin, perfPoint } from './perfMark'
66 : import type { PerfSpan } from './perfMark'
67 :
68 : /**
69 : * Startup past this is abnormal by any reading — the slowest healthy cloud
70 : * load profiled is ~5 s, and the reports that prompted this instrumentation
71 : * were minutes.
72 : */
73 2356 : const BOOTSTRAP_WATCHDOG_MS = 30_000
74 :
75 0 : function toRumTimingName(name: string): string {
76 0 : return name.replace(/\//g, '.')
77 : }
78 :
79 : const BOOTSTRAP_PHASES = {
80 : startup: [
81 : 'entry-modules',
82 : 'remote-config',
83 : 'telemetry-init',
84 : 'local-auth',
85 : 'firebase-init',
86 : 'router',
87 : 'sentry-init'
88 : ],
89 : 'auth-gate': [
90 : 'initialized',
91 : 'identity',
92 : 'sign-in',
93 : 'user-store',
94 : 'needs-login'
95 : ],
96 : bootstrap: [
97 : 'settings',
98 : 'workflows',
99 : 'i18n',
100 : 'graph-view-chunk',
101 : 'new-user',
102 : 'extensions-load',
103 : 'extensions-load-core',
104 : 'extensions-load-custom',
105 : 'deployment-pick',
106 : 'sync-workflows',
107 : 'extensions-init',
108 : 'object-info',
109 : 'extensions',
110 : 'vue-node-defs',
111 : 'extensions-setup',
112 : 'workflow-restore'
113 : ]
114 : } as const satisfies Record<string, readonly string[]>
115 :
116 : type BootstrapPhaseNamespace = keyof typeof BOOTSTRAP_PHASES
117 :
118 : type BootstrapSubphase<Namespace extends BootstrapPhaseNamespace> =
119 : (typeof BOOTSTRAP_PHASES)[Namespace][number]
120 :
121 : export type BootstrapPhase = {
122 : [
123 : Namespace in BootstrapPhaseNamespace
124 : ]: `${Namespace}/${BootstrapSubphase<Namespace>}`
125 : }[BootstrapPhaseNamespace]
126 :
127 2355 : function untracedMs(rows: BootstrapPhaseTiming[], totalMs: number): number {
128 2355 : let tracedMs = 0
129 2355 : let coveredUntil = 0
130 2355 : for (const { startMs, durationMs } of rows) {
131 54165 : const endMs = Math.min(startMs + durationMs, totalMs)
132 54165 : tracedMs += Math.max(0, endMs - Math.max(startMs, coveredUntil))
133 54165 : coveredUntil = Math.max(coveredUntil, endMs)
134 : }
135 2355 : return totalMs - tracedMs
136 : }
137 :
138 2356 : class HiddenTimeMeter {
139 : private _hiddenMs = 0
140 : private _hiddenSince: number | undefined
141 : private _listening = new AbortController()
142 :
143 : constructor() {
144 2356 : if (typeof document === 'undefined') return
145 2356 : const buffered = performance.getEntriesByType('visibility-state')
146 2356 : if (buffered.length > 0) {
147 2356 : for (const entry of buffered)
148 2356 : this._record(entry.name === 'hidden', entry.startTime)
149 0 : } else if (document.visibilityState === 'hidden') {
150 0 : this._hiddenSince = 0
151 : }
152 2356 : document.addEventListener(
153 : 'visibilitychange',
154 : () =>
155 : this._record(document.visibilityState === 'hidden', performance.now()),
156 : { signal: this._listening.signal }
157 : )
158 : }
159 :
160 : private _record(hidden: boolean, atMs: number): void {
161 2356 : if (hidden) {
162 0 : this._hiddenSince ??= atMs
163 2356 : } else if (this._hiddenSince !== undefined) {
164 0 : this._hiddenMs += atMs - this._hiddenSince
165 0 : this._hiddenSince = undefined
166 : }
167 : }
168 :
169 : stop(): void {
170 2355 : this._listening.abort()
171 : }
172 :
173 : hiddenMsUntil(nowMs: number): number {
174 2355 : const open = this._hiddenSince === undefined ? 0 : nowMs - this._hiddenSince
175 2355 : return Math.round(this._hiddenMs + Math.max(0, open))
176 : }
177 : }
178 :
179 : export interface BootstrapPhaseTiming {
180 : name: BootstrapPhase
181 : startMs: number
182 : durationMs: number
183 : }
184 :
185 2356 : export class BootstrapTracer {
186 : private _spans = new Map<BootstrapPhase, PerfSpan>()
187 : private _timings: BootstrapPhaseTiming[] = []
188 : private _completed = false
189 : private _outcome: 'completed' | 'failed' | undefined
190 : private _timeoutReported = false
191 : private _watchdog: ReturnType<typeof setTimeout> | undefined
192 : private _hidden = new HiddenTimeMeter()
193 :
194 : /**
195 : * Start timing a phase. Returns a handle; call `.stop()` when complete.
196 : * Calling startPhase for the same phase twice is safe (second call is a no-op
197 : * and returns the existing span's handle).
198 : */
199 : startPhase(phase: BootstrapPhase): { stop: () => number } {
200 51810 : if (this._spans.has(phase)) {
201 : // Already running — return a no-op handle so callers don't have to guard
202 0 : return { stop: () => 0 }
203 : }
204 51810 : this._spans.set(phase, perfMark(phase))
205 51810 : return { stop: () => this._stopPhase(phase) }
206 : }
207 :
208 : private _stopPhase(phase: BootstrapPhase): number {
209 51810 : const span = this._spans.get(phase)
210 51810 : if (!span) return 0
211 51810 : this._spans.delete(phase)
212 51810 : const durationMs = span.stop()
213 51810 : this._publishRumTiming(phase)
214 51810 : this._timings.push({
215 : name: phase,
216 : startMs: Math.round(span.startMs),
217 : durationMs: Math.round(durationMs)
218 : })
219 51810 : return durationMs
220 : }
221 :
222 : /**
223 : * Time `work` and record the phase whether it resolves or rejects, then
224 : * re-settle exactly as `work` did.
225 : *
226 : * A phase stopped only on the success path goes unrecorded precisely when
227 : * startup went wrong, so the surviving sessions are the fast ones and every
228 : * percentile reads better than reality.
229 : */
230 : async settle<T>(phase: BootstrapPhase, work: () => Promise<T>): Promise<T> {
231 44745 : const span = this.startPhase(phase)
232 44745 : try {
233 44745 : return await work()
234 : } finally {
235 44745 : span.stop()
236 : }
237 : }
238 :
239 : /**
240 : * Record a phase that began at the navigation time origin, such as the
241 : * static module graph evaluated before any startup code can run.
242 : */
243 : recordSinceTimeOrigin(phase: BootstrapPhase): void {
244 2355 : const durationMs = perfMeasureSinceOrigin(phase)
245 2355 : this._publishRumTiming(phase)
246 2355 : this._timings.push({
247 : name: phase,
248 : startMs: 0,
249 : durationMs: Math.round(durationMs)
250 : })
251 : }
252 :
253 : /**
254 : * Record a named milestone with no duration (e.g. "app shell mounted").
255 : */
256 : milestone(name: string): void {
257 4710 : const milestoneName = `bootstrap/milestone/${name}`
258 4710 : perfPoint(milestoneName)
259 4710 : this._publishRumTiming(milestoneName)
260 : }
261 :
262 : private _publishRumTiming(name: string): void {
263 58875 : getDatadogRum()?.addTiming(toRumTimingName(name))
264 : }
265 :
266 : /**
267 : * Completed phases, in the order they started.
268 : *
269 : * Built from what this tracer measured rather than re-scanned out of
270 : * `performance.getEntriesByType('measure')`: that buffer is shared with
271 : * every other measure in the app and can be cleared by anyone.
272 : */
273 : summary(): BootstrapPhaseTiming[] {
274 2355 : return [...this._timings].sort((a, b) => a.startMs - b.startMs)
275 : }
276 :
277 : /**
278 : * `timed_out` is not terminal: startup can still finish after the watchdog
279 : * fires, and the outcome then becomes `completed` or `failed`.
280 : */
281 : get outcome(): 'completed' | 'failed' | 'timed_out' | 'pending' {
282 0 : return this._outcome ?? (this._timeoutReported ? 'timed_out' : 'pending')
283 : }
284 :
285 : /**
286 : * Close out startup: stop any phase still open, emit one aggregate event
287 : * carrying total wall-clock and the per-phase breakdown, and log the trace
288 : * locally. Idempotent — only the first call reports.
289 : */
290 : complete(outcome: 'completed' | 'failed' = 'completed'): void {
291 2355 : if (this._completed) return
292 2355 : this._completed = true
293 2355 : this._outcome = outcome
294 2355 : clearTimeout(this._watchdog)
295 2355 : for (const phase of Array.from(this._spans.keys())) this._stopPhase(phase)
296 2355 : this._report(outcome)
297 2355 : this._hidden.stop()
298 : }
299 :
300 : /**
301 : * Report startup as still running if it has not completed by `deadlineMs`.
302 : *
303 : * A session that hangs never reaches `complete()`, so without this the loads
304 : * users actually report — the ones that spin for minutes — are the only ones
305 : * that produce no data at all. The watchdog row does not end the trace: the
306 : * terminal row still follows if startup eventually finishes.
307 : */
308 : armWatchdog(deadlineMs = BOOTSTRAP_WATCHDOG_MS): void {
309 2355 : clearTimeout(this._watchdog)
310 2355 : if (this._completed || this._timeoutReported) return
311 2355 : this._watchdog = setTimeout(() => {
312 0 : if (this._completed || this._timeoutReported) return
313 0 : this._timeoutReported = true
314 0 : this._report('timed_out', [...this._spans.keys()])
315 : }, deadlineMs)
316 : }
317 :
318 : suspendWatchdog(): void {
319 0 : clearTimeout(this._watchdog)
320 0 : this._watchdog = undefined
321 : }
322 :
323 : resumeWatchdog(): void {
324 2355 : if (!this._completed && this._watchdog === undefined) this.armWatchdog()
325 : }
326 :
327 : private _report(
328 : outcome: BootstrapCompleteMetadata['outcome'],
329 : pending?: BootstrapPhase[]
330 : ): void {
331 2355 : try {
332 2355 : const totalMs = Math.round(performance.now())
333 2355 : const rows = this.summary()
334 2355 : const metadata: BootstrapCompleteMetadata = {
335 : total_ms: totalMs,
336 : outcome,
337 : phase_count: rows.length,
338 : untraced_ms: untracedMs(rows, totalMs),
339 : hidden_ms: this._hidden.hiddenMsUntil(totalMs),
340 : phases: Object.fromEntries(rows.map((r) => [r.name, r.durationMs])),
341 : ...(pending?.length ? { pending } : {})
342 : }
343 2355 : getDatadogRum()?.addAction(TelemetryEvents.BOOTSTRAP_COMPLETE, metadata)
344 2355 : useTelemetry()?.trackBootstrapComplete(metadata)
345 2355 : if (outcome === 'completed') getDatadogRum()?.setViewLoadingTime()
346 2355 : this._logSummary(rows, totalMs)
347 : } catch {
348 0 : return
349 : }
350 : }
351 :
352 : /**
353 : * Log the startup summary to the console in a compact table.
354 : * Only emits in dev or when DEBUG_PERF is set.
355 : */
356 : private _logSummary(rows: BootstrapPhaseTiming[], totalMs: number): void {
357 2355 : if (!import.meta.env.DEV && localStorage.getItem('DEBUG_PERF') !== 'true')
358 2355 : return
359 0 : if (rows.length === 0) return
360 0 : console.warn(
361 : '[Bootstrap] Startup trace\n' +
362 : rows
363 : .map((r) => ` ${r.startMs}ms ${r.name}: ${r.durationMs}ms`)
364 : .join('\n') +
365 : `\n Total wall-clock: ${totalMs}ms` +
366 : `\n Untraced: ${untracedMs(rows, totalMs)}ms`
367 : )
368 : }
369 : }
370 :
371 : /** Singleton tracer — import and use directly. */
372 2356 : export const bootstrapTracer = new BootstrapTracer()
|