Line data Source code
1 : import { delay } from 'es-toolkit'
2 : import { shallowRef } from 'vue'
3 :
4 : import { reportError } from '@comfyorg/telemetry/reportError'
5 : import type { WorkspaceDeploymentCpuServer } from '@/platform/workspace/api/workspaceApi'
6 : import { useTeamWorkspaceStore } from '@/platform/workspace/stores/teamWorkspaceStore'
7 : import { WorkspaceApiError } from '@/platform/remote/ingest/ingestApiError'
8 : import { workspaceApi } from '@/platform/workspace/api/workspaceApi'
9 :
10 2355 : const CPU_SERVER_ASK_EVERY_MS = 5000
11 2355 : const CPU_SERVER_ASK_FOR_MS = 30 * 60_000
12 2355 : const CPU_SERVER_READ_ERROR_TYPE = 'cpu_server_step_read_failed'
13 2355 : const CPU_SERVER_UNKNOWN = 'cpu_server_unknown'
14 :
15 : type CpuServerAnswer = WorkspaceDeploymentCpuServer['cpu_server'] | 'unknown'
16 :
17 : export type CpuServerStep = 'creating' | 'building' | 'starting'
18 :
19 : export interface CpuServerReading {
20 : answer: CpuServerAnswer
21 : error?: unknown
22 : }
23 :
24 0 : function readingOfFailure(error: unknown): CpuServerReading {
25 0 : if (!(error instanceof WorkspaceApiError)) return { answer: 'unknown', error }
26 0 : if (error.status === 404) return { answer: 'none' }
27 0 : if (error.status === 502 && error.code === CPU_SERVER_UNKNOWN)
28 0 : return { answer: 'unknown' }
29 0 : return { answer: 'unknown', error }
30 : }
31 :
32 : /** Asks which step the CPU server of the current workspace's deployment is on; an answer that outlived its workspace is `unknown`. */
33 0 : export async function askCpuServer(): Promise<CpuServerReading> {
34 0 : const workspace = useTeamWorkspaceStore()
35 0 : const { workspaceId, workspaceTransitionGeneration: askedIn } = workspace
36 0 : if (!workspaceId) return { answer: 'none' }
37 0 : let reading: CpuServerReading
38 0 : try {
39 0 : const { cpu_server } =
40 : await workspaceApi.getDeploymentCpuServer(workspaceId)
41 0 : reading = { answer: cpu_server }
42 : } catch (error) {
43 0 : reading = readingOfFailure(error)
44 : }
45 0 : return workspace.workspaceTransitionGeneration === askedIn
46 : ? reading
47 : : { answer: 'unknown' }
48 : }
49 :
50 : /** Whether the CPU server does not exist yet, so the routes its packs serve fail until it does. */
51 0 : export function isBeingSetUp(reading: CpuServerReading | undefined): boolean {
52 0 : return reading?.answer === 'creating' || reading?.answer === 'building'
53 : }
54 :
55 0 : function stepAfter(
56 : step: CpuServerStep | undefined,
57 : answer: CpuServerAnswer
58 : ): CpuServerStep | undefined {
59 0 : switch (answer) {
60 : case 'creating':
61 : case 'building':
62 0 : return answer
63 : case 'created':
64 0 : return 'starting'
65 : case 'none':
66 0 : return undefined
67 : case 'unknown':
68 0 : return step
69 : }
70 : }
71 :
72 0 : function stepRead(
73 : step: CpuServerStep | undefined,
74 : answer: CpuServerAnswer,
75 : onUnreadable: () => void
76 : ): CpuServerStep | undefined {
77 0 : if (answer === 'unknown' && step !== undefined) onUnreadable()
78 0 : return stepAfter(step, answer)
79 : }
80 :
81 0 : async function waitUnlessStopped(
82 : ms: number,
83 : signal: AbortSignal
84 : ): Promise<boolean> {
85 0 : try {
86 0 : await delay(ms, { signal })
87 0 : return true
88 : } catch (error) {
89 0 : if (signal.aborted) return false
90 0 : throw error
91 : }
92 : }
93 :
94 0 : function reportReadFailure(error: unknown) {
95 0 : reportError(error, {
96 : errorType: CPU_SERVER_READ_ERROR_TYPE,
97 : surface: 'platform',
98 : level: 'warning'
99 : })
100 : }
101 :
102 0 : function unlessStopped<T>(
103 : work: Promise<T>,
104 : signal: AbortSignal,
105 : stopAt: number
106 : ): Promise<T | undefined> {
107 0 : if (signal.aborted || Date.now() >= stopAt) {
108 0 : work.catch(() => {})
109 0 : return Promise.resolve(undefined)
110 : }
111 0 : return new Promise((resolve, reject) => {
112 0 : const deadline = setTimeout(stop, stopAt - Date.now())
113 0 : signal.addEventListener('abort', stop, { once: true })
114 0 : work.then(
115 : (value) => {
116 0 : release()
117 0 : resolve(value)
118 : },
119 : (error: unknown) => {
120 0 : release()
121 0 : reject(error)
122 : }
123 : )
124 :
125 0 : function stop() {
126 0 : release()
127 0 : resolve(undefined)
128 : }
129 :
130 0 : function release() {
131 0 : clearTimeout(deadline)
132 0 : signal.removeEventListener('abort', stop)
133 : }
134 : })
135 : }
136 :
137 0 : async function askUntilSettled(
138 : firstReading: Promise<CpuServerReading>,
139 : signal: AbortSignal,
140 : step: { value: CpuServerStep | undefined },
141 : onUnreadable: () => void
142 : ) {
143 0 : const stopAt = Date.now() + CPU_SERVER_ASK_FOR_MS
144 0 : let asking = firstReading
145 0 : let failureReported = false
146 0 : for (;;) {
147 0 : const reading = await unlessStopped(asking, signal, stopAt)
148 0 : if (!reading) break
149 0 : if (!failureReported && reading.error !== undefined) {
150 0 : reportReadFailure(reading.error)
151 0 : failureReported = true
152 : }
153 0 : step.value = stepRead(step.value, reading.answer, onUnreadable)
154 0 : if (reading.answer === 'created' || reading.answer === 'none') return
155 0 : if (Date.now() + CPU_SERVER_ASK_EVERY_MS >= stopAt) break
156 0 : if (!(await waitUnlessStopped(CPU_SERVER_ASK_EVERY_MS, signal))) break
157 0 : asking = askCpuServer()
158 : }
159 0 : step.value = undefined
160 : }
161 :
162 : export interface CpuServerWatch {
163 : step: () => CpuServerStep | undefined
164 : ready: Promise<void>
165 : }
166 :
167 : /** Follows the CPU server's step from `firstReading`, asking every five seconds until it exists or will not come, for at most half an hour or until `signal` aborts; `ready` settles then, or once a step it named can no longer be read. */
168 0 : export function watchCpuServer(
169 : firstReading: Promise<CpuServerReading>,
170 : signal: AbortSignal
171 : ): CpuServerWatch {
172 0 : const step = shallowRef<CpuServerStep | undefined>()
173 : const ready = new Promise<void>((resolve, reject) => {
174 0 : void askUntilSettled(firstReading, signal, step, resolve).then(
175 : resolve,
176 : reject
177 : )
178 : })
179 0 : return { step: () => step.value, ready }
180 : }
|