blob: 2de96de9ace749156a9230ee0ec2a773b87deb3d [file] [log] [blame]
giob7df27f2026-07-28 10:36:17 +04001package e2e
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "io"
8 "net"
9 "net/http"
10 "os"
11 "os/exec"
12 "strings"
13 "sync"
14 "time"
15)
16
17type processSignal int
18
19const (
20 processTerminate processSignal = iota
21 processKill
22)
23
24type supervisedProcess struct {
25 name string
26 cmd *exec.Cmd
27 logPath string
28 logFile *os.File
29 pgid int
30 completed chan struct{}
31 resultMu sync.RWMutex
32 result error
33 stop sync.Once
34 stopErr error
35 redactions []string
36}
37
38func startProcess(name, logPath, dir, executable string, args ...string) (*supervisedProcess, error) {
39 logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
40 if err != nil {
41 return nil, err
42 }
43 cmd := exec.Command(executable, args...)
44 cmd.Dir = dir
45 cmd.Stdout = logFile
46 cmd.Stderr = logFile
47 if err := configureOwnedProcess(cmd); err != nil {
48 _ = logFile.Close()
49 return nil, fmt.Errorf("configure %s process ownership: %w", name, err)
50 }
51 if err := cmd.Start(); err != nil {
52 _ = logFile.Close()
53 return nil, fmt.Errorf("start %s: %w", name, err)
54 }
55 p := &supervisedProcess{name: name, cmd: cmd, logPath: logPath, logFile: logFile, pgid: cmd.Process.Pid, completed: make(chan struct{})}
56 go func() {
57 err := cmd.Wait()
58 _ = logFile.Close()
59 p.resultMu.Lock()
60 p.result = err
61 p.resultMu.Unlock()
62 close(p.completed)
63 }()
64 return p, nil
65}
66
67func (p *supervisedProcess) alive() bool {
68 if p == nil {
69 return false
70 }
71 select {
72 case <-p.completed:
73 return false
74 default:
75 return true
76 }
77}
78
79func (p *supervisedProcess) completion() error {
80 p.resultMu.RLock()
81 defer p.resultMu.RUnlock()
82 return p.result
83}
84
85func (p *supervisedProcess) stopAndWait(timeout time.Duration) error {
86 if p == nil {
87 return nil
88 }
89 p.stop.Do(func() {
90 // Signal the group even when the leader has already exited: descendants may
91 // still own the process group and are part of this harness's ownership.
92 if err := signalOwnedProcessGroup(p.pgid, processTerminate); err != nil && !errors.Is(err, os.ErrProcessDone) {
93 p.stopErr = fmt.Errorf("terminate %s process group: %w", p.name, err)
94 }
95 deadline := time.Now().Add(timeout)
96 for ownedProcessGroupAlive(p.pgid) && time.Now().Before(deadline) {
97 time.Sleep(10 * time.Millisecond)
98 }
99 if ownedProcessGroupAlive(p.pgid) {
100 if err := signalOwnedProcessGroup(p.pgid, processKill); err != nil && !errors.Is(err, os.ErrProcessDone) && p.stopErr == nil {
101 p.stopErr = fmt.Errorf("kill %s process group: %w", p.name, err)
102 }
103 }
104
105 postKill := time.NewTimer(2 * time.Second)
106 defer postKill.Stop()
107 select {
108 case <-p.completed:
109 case <-postKill.C:
110 if p.stopErr == nil {
111 p.stopErr = fmt.Errorf("wait for %s leader after process-group kill: timeout", p.name)
112 }
113 }
114 groupDeadline := time.Now().Add(2 * time.Second)
115 for ownedProcessGroupAlive(p.pgid) && time.Now().Before(groupDeadline) {
116 time.Sleep(10 * time.Millisecond)
117 }
118 if ownedProcessGroupAlive(p.pgid) && p.stopErr == nil {
119 p.stopErr = fmt.Errorf("%s process group %d survived cleanup", p.name, p.pgid)
120 }
121 // A non-zero leader exit belongs to startup/readiness diagnostics, not
122 // cleanup. Once the owned group is gone, explicit cleanup succeeded.
123 })
124 return p.stopErr
125}
126
127func (p *supervisedProcess) exited() (bool, error) {
128 select {
129 case <-p.completed:
130 return true, p.completion()
131 default:
132 return false, nil
133 }
134}
135
136func (p *supervisedProcess) logTail(limit int64) string {
137 data, err := os.ReadFile(p.logPath)
138 if err != nil {
139 return fmt.Sprintf("<read log: %v>", err)
140 }
141 if int64(len(data)) > limit {
142 data = data[len(data)-int(limit):]
143 }
144 text := boundedText(data, int(limit))
145 for _, secret := range p.redactions {
146 if secret != "" {
147 text = strings.ReplaceAll(text, secret, "[REDACTED]")
148 }
149 }
150 return text
151}
152
153func waitHTTPReady(ctx context.Context, p *supervisedProcess, endpoints ...string) error {
154 client := &http.Client{Timeout: 750 * time.Millisecond}
155 defer client.CloseIdleConnections()
156 for {
157 allReady := true
158 for _, endpoint := range endpoints {
159 req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
160 if err != nil {
161 return err
162 }
163 resp, err := client.Do(req)
164 if err != nil {
165 allReady = false
166 break
167 }
168 _, _ = io.CopyN(io.Discard, resp.Body, 4096)
169 _ = resp.Body.Close()
170 if resp.StatusCode < 200 || resp.StatusCode >= 300 {
171 allReady = false
172 break
173 }
174 }
175 if allReady {
176 return nil
177 }
178 if exited, err := p.exited(); exited {
179 return fmt.Errorf("%s exited during readiness: %v\nlog tail:\n%s", p.name, err, p.logTail(8192))
180 }
181 select {
182 case <-ctx.Done():
183 return fmt.Errorf("%s readiness: %w\nlog tail:\n%s", p.name, ctx.Err(), p.logTail(8192))
184 case <-time.After(100 * time.Millisecond):
185 }
186 }
187}
188
189func waitTCPReady(ctx context.Context, p *supervisedProcess, addresses ...string) error {
190 pending := append([]string(nil), addresses...)
191 for len(pending) > 0 {
192 next := pending[:0]
193 for _, address := range pending {
194 conn, err := (&net.Dialer{Timeout: 500 * time.Millisecond}).DialContext(ctx, "tcp", address)
195 if err == nil {
196 _ = conn.Close()
197 continue
198 }
199 next = append(next, address)
200 }
201 pending = next
202 if len(pending) == 0 {
203 return nil
204 }
205 if exited, exitErr := p.exited(); exited {
206 return fmt.Errorf("%s exited during readiness: %v\nlog tail:\n%s", p.name, exitErr, p.logTail(8192))
207 }
208 select {
209 case <-ctx.Done():
210 return fmt.Errorf("%s readiness for %s: %w\nlog tail:\n%s", p.name, strings.Join(pending, ", "), ctx.Err(), p.logTail(8192))
211 case <-time.After(100 * time.Millisecond):
212 }
213 }
214 return nil
215}
216
217func boundedText(data []byte, limit int) string {
218 if len(data) > limit {
219 data = data[len(data)-limit:]
220 }
221 return strings.ToValidUTF8(string(data), "�")
222}