blob: 342a3f3d835007f6fcb4c8929b3dfc898a5e7477 [file] [log] [blame]
gioa1efbad2025-05-21 07:16:45 +00001import { z } from "zod";
2
gio78a22882025-07-01 18:56:01 +00003const LogItemSchema = z.object({
4 runId: z.string(),
5 timestampMilli: z.number(),
6 commit: z.string().optional(),
7 contents: z.preprocess((val) => {
8 if (typeof val === "string") {
9 return Buffer.from(val, "base64").toString("utf-8");
10 }
11 throw new Error("Log item contents is not a string");
12 }, z.string()),
13});
14
15export type LogItem = z.infer<typeof LogItemSchema>;
16
17const LogItemsSchema = z.array(LogItemSchema);
18
gioa1efbad2025-05-21 07:16:45 +000019export const WorkerSchema = z.object({
20 id: z.string(),
21 service: z.string(),
22 address: z.string().url(),
23 status: z.optional(
24 z.object({
gioa70535a2025-07-02 15:50:25 +000025 commit: z
26 .optional(
27 z.object({
28 hash: z.string(),
29 message: z.string(),
30 }),
31 )
32 .nullable(),
gioa1efbad2025-05-21 07:16:45 +000033 commands: z.optional(
34 z.array(
35 z.object({
36 command: z.string(),
37 state: z.string(),
38 }),
39 ),
40 ),
41 }),
42 ),
gio78a22882025-07-01 18:56:01 +000043 logs: LogItemsSchema.optional(),
gioa1efbad2025-05-21 07:16:45 +000044});
45
46export type Worker = z.infer<typeof WorkerSchema>;
47
48class ServiceMonitor {
49 private workers: Map<string, string> = new Map();
gioa1efbad2025-05-21 07:16:45 +000050 private statuses: Map<string, Worker["status"]> = new Map();
gio166d9922025-07-07 17:30:21 +000051 private lastPings: Map<string, number> = new Map();
gioa1efbad2025-05-21 07:16:45 +000052
53 constructor(public readonly serviceName: string) {}
54
gio40c0c992025-07-02 13:18:05 +000055 registerWorker(workerId: string, workerAddress: string, workerStatus?: Worker["status"]): void {
gioa1efbad2025-05-21 07:16:45 +000056 this.workers.set(workerId, workerAddress);
gioa1efbad2025-05-21 07:16:45 +000057 if (workerStatus) {
58 this.statuses.set(workerId, workerStatus);
59 }
gio166d9922025-07-07 17:30:21 +000060 this.lastPings.set(workerId, Date.now());
gioa1efbad2025-05-21 07:16:45 +000061 }
62
63 getWorkerAddress(workerId: string): string | undefined {
64 return this.workers.get(workerId);
65 }
66
gioa1efbad2025-05-21 07:16:45 +000067 getWorkerStatus(workerId: string): Worker["status"] | undefined {
68 return this.statuses.get(workerId);
69 }
70
gioa1efbad2025-05-21 07:16:45 +000071 getAllStatuses(): Map<string, Worker["status"]> {
72 return new Map(this.statuses);
73 }
74
75 getWorkerAddresses(): string[] {
76 return Array.from(this.workers.values());
77 }
78
79 getWorkerIds(): string[] {
80 return Array.from(this.workers.keys());
81 }
82
gio918780d2025-05-22 08:24:41 +000083 async reloadWorker(workerId: string): Promise<void> {
84 const workerAddress = this.workers.get(workerId);
85 if (!workerAddress) {
86 throw new Error(`Worker ${workerId} not found in service ${this.serviceName}`);
87 }
88 try {
89 const response = await fetch(`${workerAddress}/update`, { method: "POST" });
90 if (!response.ok) {
91 throw new Error(
92 `Failed to trigger reload for worker ${workerId} at ${workerAddress}: ${response.statusText}`,
93 );
94 }
95 console.log(`Reload triggered for worker ${workerId} in service ${this.serviceName}`);
96 } catch (error) {
97 console.error(`Error reloading worker ${workerId} in service ${this.serviceName}:`, error);
98 throw error; // Re-throw to be caught by ProjectMonitor
99 }
100 }
gio577d2342025-07-03 12:50:18 +0000101
102 async terminateWorker(workerId: string): Promise<void> {
103 const workerAddress = this.workers.get(workerId);
104 if (!workerAddress) {
105 throw new Error(`Worker ${workerId} not found in service ${this.serviceName}`);
106 }
107 try {
108 const response = await fetch(`${workerAddress}/quitquitquit`, { method: "POST" });
109 if (!response.ok) {
110 throw new Error(`Failed to terminate worker ${workerId} at ${workerAddress}: ${response.statusText}`);
111 }
112 console.log(`Terminated worker ${workerId} in service ${this.serviceName}`);
113 } catch (error) {
114 console.error(`Error terminating worker ${workerId} in service ${this.serviceName}:`, error);
115 throw error; // Re-throw to be caught by ProjectMonitor
116 }
117 }
gio166d9922025-07-07 17:30:21 +0000118
119 cleanupWorkers(now: number) {
120 const toDelete: string[] = [];
121 this.workers.forEach((_, workerId) => {
122 if (now - (this.lastPings.get(workerId) ?? 0) > 1000 * 60) {
123 toDelete.push(workerId);
124 }
125 });
126 for (const workerId of toDelete) {
127 this.workers.delete(workerId);
128 this.statuses.delete(workerId);
129 this.lastPings.delete(workerId);
130 }
131 }
gioa1efbad2025-05-21 07:16:45 +0000132}
133
134export class ProjectMonitor {
135 private serviceMonitors: Map<string, ServiceMonitor> = new Map();
136
137 constructor() {}
138
139 registerWorker(workerData: Worker): void {
140 let serviceMonitor = this.serviceMonitors.get(workerData.service);
141 if (!serviceMonitor) {
142 serviceMonitor = new ServiceMonitor(workerData.service);
143 this.serviceMonitors.set(workerData.service, serviceMonitor);
144 }
gio40c0c992025-07-02 13:18:05 +0000145 serviceMonitor.registerWorker(workerData.id, workerData.address, workerData.status);
gioa1efbad2025-05-21 07:16:45 +0000146 }
147
148 getWorkerAddresses(): string[] {
149 let allAddresses: string[] = [];
150 for (const serviceMonitor of this.serviceMonitors.values()) {
151 allAddresses = allAddresses.concat(serviceMonitor.getWorkerAddresses());
152 }
153 return Array.from(new Set(allAddresses));
154 }
155
gioa1efbad2025-05-21 07:16:45 +0000156 getAllServiceNames(): string[] {
157 return Array.from(this.serviceMonitors.keys());
158 }
159
gioa1efbad2025-05-21 07:16:45 +0000160 getServiceMonitor(serviceName: string): ServiceMonitor | undefined {
161 return this.serviceMonitors.get(serviceName);
162 }
163
164 getWorkerStatusesForService(serviceName: string): Map<string, Worker["status"]> {
165 const serviceMonitor = this.serviceMonitors.get(serviceName);
166 if (serviceMonitor) {
167 return serviceMonitor.getAllStatuses();
168 }
169 return new Map();
170 }
gio918780d2025-05-22 08:24:41 +0000171
172 async reloadWorker(serviceName: string, workerId: string): Promise<void> {
173 const serviceMonitor = this.serviceMonitors.get(serviceName);
174 if (!serviceMonitor) {
175 throw new Error(`Service ${serviceName} not found`);
176 }
177 await serviceMonitor.reloadWorker(workerId);
178 }
gio577d2342025-07-03 12:50:18 +0000179
180 async terminateWorker(serviceName: string, workerId: string): Promise<void> {
181 const serviceMonitor = this.serviceMonitors.get(serviceName);
182 if (!serviceMonitor) {
183 throw new Error(`Service ${serviceName} not found`);
184 }
185 await serviceMonitor.terminateWorker(workerId);
186 }
gio166d9922025-07-07 17:30:21 +0000187
188 cleanupWorkers(now: number) {
189 this.serviceMonitors.forEach((serviceMonitor) => {
190 serviceMonitor.cleanupWorkers(now);
191 });
192 }
gioa1efbad2025-05-21 07:16:45 +0000193}