Installer: deallocate ports upon app uninstall
Change-Id: I19298537fed02de03a9e74fa351cf23f733de699
diff --git a/core/installer/app.go b/core/installer/app.go
index e443948..0eaa27d 100644
--- a/core/installer/app.go
+++ b/core/installer/app.go
@@ -83,6 +83,7 @@
type PortForward struct {
Allocator string `json:"allocator"`
ReserveAddr string `json:"reservator"`
+ RemoveAddr string `json:"deallocator"`
Protocol string `json:"protocol"`
SourcePort int `json:"sourcePort"`
TargetService string `json:"targetService"`
diff --git a/core/installer/app_configs/app_base.cue b/core/installer/app_configs/app_base.cue
index 428fbc8..322f736 100644
--- a/core/installer/app_configs/app_base.cue
+++ b/core/installer/app_configs/app_base.cue
@@ -33,6 +33,7 @@
domain: string
allocatePortAddr: string
reservePortAddr: string
+ deallocatePortAddr: string
}
#Image: {
@@ -84,6 +85,7 @@
#PortForward: {
allocator: string
reservator: string
+ deallocator: string
protocol: "TCP" | "UDP" | *"TCP"
sourcePort: int
serviceName: string
diff --git a/core/installer/app_manager.go b/core/installer/app_manager.go
index 4f01d46..cad18eb 100644
--- a/core/installer/app_manager.go
+++ b/core/installer/app_manager.go
@@ -152,7 +152,14 @@
SourcePort int `json:"sourcePort"`
TargetService string `json:"targetService"`
TargetPort int `json:"targetPort"`
- Secret string `json:"secret"`
+ Secret string `json:"secret,omitempty"`
+}
+
+type removePortReq struct {
+ Protocol string `json:"protocol"`
+ SourcePort int `json:"sourcePort"`
+ TargetService string `json:"targetService"`
+ TargetPort int `json:"targetPort"`
}
type reservePortResp struct {
@@ -209,12 +216,41 @@
return err
}
if resp.StatusCode != http.StatusOK {
- return fmt.Errorf("Could not allocate port %d, status code: %d", p.SourcePort, resp.StatusCode)
+ var r bytes.Buffer
+ io.Copy(&r, resp.Body)
+ return fmt.Errorf("Could not allocate port %d, status code %d, message: %s", p.SourcePort, resp.StatusCode, r.String())
}
}
return nil
}
+func closePorts(ports []PortForward) error {
+ var retErr error
+ for _, p := range ports {
+ var buf bytes.Buffer
+ req := removePortReq{
+ Protocol: p.Protocol,
+ SourcePort: p.SourcePort,
+ TargetService: p.TargetService,
+ TargetPort: p.TargetPort,
+ }
+ if err := json.NewEncoder(&buf).Encode(req); err != nil {
+ retErr = err
+ continue
+ }
+ resp, err := http.Post(p.RemoveAddr, "application/json", &buf)
+ if err != nil {
+ retErr = err
+ continue
+ }
+ if resp.StatusCode != http.StatusOK {
+ retErr = fmt.Errorf("Could not deallocate port %d, status code: %d", p.SourcePort, resp.StatusCode)
+ continue
+ }
+ }
+ return retErr
+}
+
func createKustomizationChain(r soft.RepoFS, path string) error {
for p := filepath.Clean(path); p != "/"; {
parent, child := filepath.Split(p)
@@ -419,6 +455,7 @@
}
// TODO(gio): add ingress-nginx to release resources
if err := openPorts(rendered.Ports, portReservations, allocators); err != nil {
+ fmt.Println(err)
return ReleaseResources{}, err
}
return ReleaseResources{
@@ -489,11 +526,11 @@
if err != nil {
return ReleaseResources{}, err
}
- localCharts, err := extractLocalCharts(m.repoIO, filepath.Join(instanceDir, "rendered.json"))
+ renderedCfg, err := readRendered(m.repoIO, filepath.Join(instanceDir, "rendered.json"))
if err != nil {
return ReleaseResources{}, err
}
- rendered, err := app.Render(config.Release, env, values, localCharts)
+ rendered, err := app.Render(config.Release, env, values, renderedCfg.LocalCharts)
if err != nil {
return ReleaseResources{}, err
}
@@ -504,8 +541,15 @@
if err := m.repoIO.Pull(); err != nil {
return err
}
- return m.repoIO.Do(func(r soft.RepoFS) (string, error) {
- r.RemoveDir(filepath.Join(m.appDirRoot, instanceId))
+ var portForward []PortForward
+ if err := m.repoIO.Do(func(r soft.RepoFS) (string, error) {
+ instanceDir := filepath.Join(m.appDirRoot, instanceId)
+ renderedCfg, err := readRendered(m.repoIO, filepath.Join(instanceDir, "rendered.json"))
+ if err != nil {
+ return "", err
+ }
+ portForward = renderedCfg.PortForward
+ r.RemoveDir(instanceDir)
kustPath := filepath.Join(m.appDirRoot, "kustomization.yaml")
kust, err := soft.ReadKustomization(r, kustPath)
if err != nil {
@@ -514,26 +558,35 @@
kust.RemoveResources(instanceId)
soft.WriteYaml(r, kustPath, kust)
return fmt.Sprintf("uninstall: %s", instanceId), nil
- })
+ }); err != nil {
+ return err
+ }
+ if err := closePorts(portForward); err != nil {
+ fmt.Println(err)
+ return err
+ }
+ return nil
}
// TODO(gio): deduplicate with cue definition in app.go, this one should be removed.
func CreateNetworks(env EnvConfig) []Network {
return []Network{
{
- Name: "Public",
- IngressClass: fmt.Sprintf("%s-ingress-public", env.InfraName),
- CertificateIssuer: fmt.Sprintf("%s-public", env.Id),
- Domain: env.Domain,
- AllocatePortAddr: fmt.Sprintf("http://port-allocator.%s-ingress-public.svc.cluster.local/api/allocate", env.InfraName),
- ReservePortAddr: fmt.Sprintf("http://port-allocator.%s-ingress-public.svc.cluster.local/api/reserve", env.InfraName),
+ Name: "Public",
+ IngressClass: fmt.Sprintf("%s-ingress-public", env.InfraName),
+ CertificateIssuer: fmt.Sprintf("%s-public", env.Id),
+ Domain: env.Domain,
+ AllocatePortAddr: fmt.Sprintf("http://port-allocator.%s-ingress-public.svc.cluster.local/api/allocate", env.InfraName),
+ ReservePortAddr: fmt.Sprintf("http://port-allocator.%s-ingress-public.svc.cluster.local/api/reserve", env.InfraName),
+ DeallocatePortAddr: fmt.Sprintf("http://port-allocator.%s-ingress-public.svc.cluster.local/api/remove", env.InfraName),
},
{
- Name: "Private",
- IngressClass: fmt.Sprintf("%s-ingress-private", env.Id),
- Domain: env.PrivateDomain,
- AllocatePortAddr: fmt.Sprintf("http://port-allocator.%s-ingress-private.svc.cluster.local/api/allocate", env.Id),
- ReservePortAddr: fmt.Sprintf("http://port-allocator.%s-ingress-private.svc.cluster.local/api/reserve", env.Id),
+ Name: "Private",
+ IngressClass: fmt.Sprintf("%s-ingress-private", env.Id),
+ Domain: env.PrivateDomain,
+ AllocatePortAddr: fmt.Sprintf("http://port-allocator.%s-ingress-private.svc.cluster.local/api/allocate", env.Id),
+ ReservePortAddr: fmt.Sprintf("http://port-allocator.%s-ingress-private.svc.cluster.local/api/reserve", env.Id),
+ DeallocatePortAddr: fmt.Sprintf("http://port-allocator.%s-ingress-private.svc.cluster.local/api/remove", env.Id),
},
}
}
@@ -703,11 +756,11 @@
if err != nil {
return ReleaseResources{}, err
}
- localCharts, err := extractLocalCharts(m.repoIO, filepath.Join(instanceDir, "rendered.json"))
+ renderedCfg, err := readRendered(m.repoIO, filepath.Join(instanceDir, "rendered.json"))
if err != nil {
return ReleaseResources{}, err
}
- rendered, err := app.Render(config.Release, env, values, localCharts)
+ rendered, err := app.Render(config.Release, env, values, renderedCfg.LocalCharts)
if err != nil {
return ReleaseResources{}, err
}
@@ -754,19 +807,20 @@
type renderedInstance struct {
LocalCharts map[string]helmv2.HelmChartTemplateSpec `json:"localCharts"`
+ PortForward []PortForward `json:"portForward"`
}
-func extractLocalCharts(fs soft.RepoFS, path string) (map[string]helmv2.HelmChartTemplateSpec, error) {
+func readRendered(fs soft.RepoFS, path string) (renderedInstance, error) {
r, err := fs.Reader(path)
if err != nil {
- return nil, err
+ return renderedInstance{}, err
}
defer r.Close()
var cfg renderedInstance
if err := json.NewDecoder(r).Decode(&cfg); err != nil {
- return nil, err
+ return renderedInstance{}, err
}
- return cfg.LocalCharts, nil
+ return cfg, nil
}
func findPortFields(scm Schema) []string {
diff --git a/core/installer/derived.go b/core/installer/derived.go
index 542f70d..e4756be 100644
--- a/core/installer/derived.go
+++ b/core/installer/derived.go
@@ -14,12 +14,13 @@
}
type Network struct {
- Name string `json:"name,omitempty"`
- IngressClass string `json:"ingressClass,omitempty"`
- CertificateIssuer string `json:"certificateIssuer,omitempty"`
- Domain string `json:"domain,omitempty"`
- AllocatePortAddr string `json:"allocatePortAddr,omitempty"`
- ReservePortAddr string `json:"reservePortAddr,omitempty"`
+ Name string `json:"name,omitempty"`
+ IngressClass string `json:"ingressClass,omitempty"`
+ CertificateIssuer string `json:"certificateIssuer,omitempty"`
+ Domain string `json:"domain,omitempty"`
+ AllocatePortAddr string `json:"allocatePortAddr,omitempty"`
+ ReservePortAddr string `json:"reservePortAddr,omitempty"`
+ DeallocatePortAddr string `json:"deallocatePortAddr,omitempty"`
}
type InfraAppInstanceConfig struct {
diff --git a/core/installer/schema.go b/core/installer/schema.go
index 8ddf73f..943ef8a 100644
--- a/core/installer/schema.go
+++ b/core/installer/schema.go
@@ -61,6 +61,7 @@
domain: string
allocatePortAddr: string
reservePortAddr: string
+ deallocatePortAddr: string
}
value: { %s }
diff --git a/core/installer/values-tmpl/dodo-app.cue b/core/installer/values-tmpl/dodo-app.cue
index e9fcebe..d60d1ea 100644
--- a/core/installer/values-tmpl/dodo-app.cue
+++ b/core/installer/values-tmpl/dodo-app.cue
@@ -56,6 +56,7 @@
portForward: [#PortForward & {
allocator: input.network.allocatePortAddr
reservator: input.network.reservePortAddr
+ deallocator: input.network.deallocatePortAddr
sourcePort: input.sshPort
serviceName: "soft-serve"
targetPort: 22
diff --git a/core/installer/values-tmpl/gerrit.cue b/core/installer/values-tmpl/gerrit.cue
index a2a88f9..060a7d2 100644
--- a/core/installer/values-tmpl/gerrit.cue
+++ b/core/installer/values-tmpl/gerrit.cue
@@ -106,6 +106,7 @@
portForward: [#PortForward & {
allocator: input.network.allocatePortAddr
reservator: input.network.reservePortAddr
+ deallocator: input.network.deallocatePortAddr
sourcePort: input.sshPort
serviceName: "gerrit-gerrit-service"
targetPort: _sshPort
diff --git a/core/installer/values-tmpl/soft-serve.cue b/core/installer/values-tmpl/soft-serve.cue
index d172b9b..8aeb49f 100644
--- a/core/installer/values-tmpl/soft-serve.cue
+++ b/core/installer/values-tmpl/soft-serve.cue
@@ -47,6 +47,7 @@
portForward: [#PortForward & {
allocator: input.network.allocatePortAddr
reservator: input.network.reservePortAddr
+ deallocator: input.network.deallocatePortAddr
sourcePort: input.sshPort
serviceName: "soft-serve"
targetPort: 22
diff --git a/core/port-allocator/main.go b/core/port-allocator/main.go
index 48ffd35..d99550b 100644
--- a/core/port-allocator/main.go
+++ b/core/port-allocator/main.go
@@ -71,8 +71,9 @@
}
func (s *server) Start() error {
- s.r.HandleFunc("/api/allocate", s.handleAllocate)
s.r.HandleFunc("/api/reserve", s.handleReserve)
+ s.r.HandleFunc("/api/allocate", s.handleAllocate)
+ s.r.HandleFunc("/api/remove", s.handleRemove)
if err := s.s.ListenAndServe(); err != nil && err != http.ErrServerClosed {
return err
}
@@ -91,6 +92,13 @@
Secret string `json:"secret"`
}
+type removeReq struct {
+ Protocol string `json:"protocol"`
+ SourcePort int `json:"sourcePort"`
+ TargetService string `json:"targetService"`
+ TargetPort int `json:"targetPort"`
+}
+
func extractAllocateReq(r io.Reader) (allocateReq, error) {
var req allocateReq
if err := json.NewDecoder(r).Decode(&req); err != nil {
@@ -103,6 +111,18 @@
return req, nil
}
+func extractRemoveReq(r io.Reader) (removeReq, error) {
+ var req removeReq
+ if err := json.NewDecoder(r).Decode(&req); err != nil {
+ return removeReq{}, err
+ }
+ req.Protocol = strings.ToLower(req.Protocol)
+ if req.Protocol != "tcp" && req.Protocol != "udp" {
+ return removeReq{}, fmt.Errorf("Unexpected protocol %s", req.Protocol)
+ }
+ return req, nil
+}
+
type reserveResp struct {
Port int `json:"port"`
Secret string `json:"secret"`
@@ -155,6 +175,15 @@
return nil
}
+func removePort(pm map[string]any, req removeReq) error {
+ sourcePortStr := strconv.Itoa(req.SourcePort)
+ if _, ok := pm[sourcePortStr]; !ok {
+ return fmt.Errorf("port %d is not open to remove", req.SourcePort)
+ }
+ delete(pm, sourcePortStr)
+ return nil
+}
+
const start = 49152
const end = 65535
@@ -266,6 +295,50 @@
}
}
+func (s *server) handleRemove(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ http.Error(w, "only post method is supported", http.StatusBadRequest)
+ return
+ }
+ req, err := extractRemoveReq(r.Body)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ s.l.Lock()
+ defer s.l.Unlock()
+ ingressRel, err := s.client.ReadRelease()
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ tcp, udp, err := extractPorts(ingressRel)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ switch req.Protocol {
+ case "tcp":
+ if err := removePort(tcp, req); err != nil {
+ http.Error(w, err.Error(), http.StatusConflict)
+ return
+ }
+ case "udp":
+ if err := removePort(udp, req); err != nil {
+ http.Error(w, err.Error(), http.StatusConflict)
+ return
+ }
+ default:
+ panic("MUST NOT REACH")
+ }
+ commitMsg := fmt.Sprintf("ingress: remove port map %d %s", req.SourcePort, req.Protocol)
+ if err := s.client.WriteRelease(ingressRel, commitMsg); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ delete(s.reserve, req.SourcePort)
+}
+
// TODO(gio): deduplicate
func createRepoClient(addr string, keyPath string) (soft.RepoIO, error) {
sshKey, err := os.ReadFile(keyPath)