env-manager: dynamically generate cidr for new env (#85)
* env-manager: allocate env cidrs dynamically
* fix: net.IP to netip.Addr conversion
* bootstrapper: generate empty env-cidrs.yaml
* fix: net.IP to netip.Addr conversion for IP pool
* infra: expose provided startIP subnet via tailscale proxy
* headscale: pass private network ip subnet to expose to api service
* dns: make ingress IP configurable
---------
Co-authored-by: Giorgi Lekveishvili <lekva@gl-mbp-m1-max.local>
diff --git a/charts/headscale/templates/headscale.yaml b/charts/headscale/templates/headscale.yaml
index fa58ab5..d483854 100644
--- a/charts/headscale/templates/headscale.yaml
+++ b/charts/headscale/templates/headscale.yaml
@@ -177,7 +177,7 @@
- headscale-api
- --port={{ .Values.api.port }}
- --config=/headscale/config/config.yaml
- - --domain={{ .Values.api.rootDomain }}
+ - --ip-subnet={{ .Values.api.ipSubnet }}
- --acls=/headscale/acls/config.hujson
volumeMounts:
- name: data
diff --git a/charts/headscale/values.yaml b/charts/headscale/values.yaml
index 20247f2..6152794 100644
--- a/charts/headscale/values.yaml
+++ b/charts/headscale/values.yaml
@@ -15,7 +15,7 @@
ipAddressPool: example-headscale
api:
port: 8585
- rootDomain: example.com
+ ipSubnet: 10.1.0.0/24
image:
repository: giolekva/headscale-api
tag: latest
diff --git a/core/headscale/main.go b/core/headscale/main.go
index 942d71a..87d245b 100644
--- a/core/headscale/main.go
+++ b/core/headscale/main.go
@@ -5,6 +5,7 @@
"flag"
"fmt"
"log"
+ "net"
"net/http"
"os"
"text/template"
@@ -15,28 +16,21 @@
var port = flag.Int("port", 3000, "Port to listen on")
var config = flag.String("config", "", "Path to headscale config")
var acls = flag.String("acls", "", "Path to the headscale acls file")
-var domain = flag.String("domain", "", "Environment domain")
+var ipSubnet = flag.String("ip-subnet", "10.1.0.0/24", "IP subnet of the private network")
// TODO(gio): make internal network cidr and proxy user configurable
const defaultACLs = `
{
"autoApprovers": {
"routes": {
- // "10.1.0.0/24": ["private-network-proxy@{{ .Domain }}"],
- "10.1.0.0/24": ["*"],
+ "{{ .ipSubnet }}": ["*"],
},
},
"acls": [
{ // Everyone has passthough access to private-network-proxy node
"action": "accept",
"src": ["*"],
- "dst": ["10.1.0.0/24:*", "private-network-proxy:0"],
- },
- ],
- "tests": [
- {
- "src": "*",
- "accept": ["10.1.0.1:80", "10.1.0.1:443"],
+ "dst": ["{{ .ipSubnet }}:*", "private-network-proxy:0"],
},
],
}
@@ -94,24 +88,28 @@
}
}
-func updateACLs(domain, acls string) error {
+func updateACLs(cidr net.IPNet, aclsPath string) error {
tmpl, err := template.New("acls").Parse(defaultACLs)
if err != nil {
return err
}
- out, err := os.Create(acls)
+ out, err := os.Create(aclsPath)
if err != nil {
return err
}
defer out.Close()
return tmpl.Execute(out, map[string]any{
- "Domain": domain,
+ "ipSubnet": cidr.String(),
})
}
func main() {
flag.Parse()
- updateACLs(*domain, *acls)
+ _, cidr, err := net.ParseCIDR(*ipSubnet)
+ if err != nil {
+ panic(err)
+ }
+ updateACLs(*cidr, *acls)
c := newClient(*config)
s := newServer(*port, c)
s.start()
diff --git a/core/installer/bootstrapper.go b/core/installer/bootstrapper.go
index 7ac86c1..71b18d4 100644
--- a/core/installer/bootstrapper.go
+++ b/core/installer/bootstrapper.go
@@ -427,6 +427,9 @@
if err := repo.WriteYaml("config.yaml", env); err != nil {
return err
}
+ if err := repo.WriteYaml("env-cidrs.yaml", EnvCIDRs{}); err != nil {
+ return err
+ }
kust := NewKustomization()
kust.AddResources(
fmt.Sprintf("%s-flux", env.Name),
diff --git a/core/installer/config.go b/core/installer/config.go
index be35735..5269512 100644
--- a/core/installer/config.go
+++ b/core/installer/config.go
@@ -1,6 +1,7 @@
package installer
import (
+ "net"
"net/netip"
)
@@ -21,6 +22,13 @@
ServiceIPs EnvServiceIPs `json:"serviceIPs"`
}
+type EnvCIDR struct {
+ Name string
+ IP net.IP
+}
+
+type EnvCIDRs []EnvCIDR
+
type Config struct {
Values Values `json:"input"` // TODO(gio): rename
}
diff --git a/core/installer/tasks/dns.go b/core/installer/tasks/dns.go
index 02d8fee..0880782 100644
--- a/core/installer/tasks/dns.go
+++ b/core/installer/tasks/dns.go
@@ -14,10 +14,10 @@
type Check func(ch Check) error
-func SetupZoneTask(env Env, st *state) Task {
+func SetupZoneTask(env Env, ingressIP net.IP, st *state) Task {
return newSequentialParentTask(
fmt.Sprintf("Setup DNS zone records for %s", env.Domain),
- CreateZoneRecords(env.Domain, st.publicIPs, env, st),
+ CreateZoneRecords(env.Domain, st.publicIPs, ingressIP, env, st),
WaitToPropagate(env.Domain, st.publicIPs),
)
}
@@ -25,6 +25,7 @@
func CreateZoneRecords(
name string,
expected []net.IP,
+ ingressIP net.IP,
env Env,
st *state,
) Task {
@@ -52,7 +53,7 @@
namespace: {{ .namespace }}
spec:
zone: {{ .zone }}
- privateIP: 10.1.0.1
+ privateIP: {{ .ingressIP }}
publicIPs:
{{ range .publicIPs }}
- {{ .String }}
@@ -85,6 +86,7 @@
"zone": env.Domain,
"dnssec": key,
"publicIPs": st.publicIPs,
+ "ingressIP": ingressIP.String(),
}); err != nil {
return err
}
diff --git a/core/installer/tasks/env.go b/core/installer/tasks/env.go
index 04b3c9a..eb537da 100644
--- a/core/installer/tasks/env.go
+++ b/core/installer/tasks/env.go
@@ -41,6 +41,7 @@
func NewCreateEnvTask(
env Env,
publicIPs []net.IP,
+ startIP net.IP,
nsCreator installer.NamespaceCreator,
repo installer.RepoIO,
) (Task, DNSZoneRef) {
@@ -55,9 +56,9 @@
[]Task{
SetupConfigRepoTask(env, &st),
NewActivateEnvTask(env, &st),
- SetupZoneTask(env, &st),
+ SetupZoneTask(env, startIP, &st),
},
- SetupInfra(env, &st)...,
+ SetupInfra(env, startIP, &st)...,
)...,
)
rctx, done := context.WithCancel(context.Background())
diff --git a/core/installer/tasks/infra.go b/core/installer/tasks/infra.go
index 39d9f50..cd7ad14 100644
--- a/core/installer/tasks/infra.go
+++ b/core/installer/tasks/infra.go
@@ -2,6 +2,7 @@
import (
"fmt"
+ "net"
"net/netip"
"github.com/miekg/dns"
@@ -9,7 +10,7 @@
"github.com/giolekva/pcloud/core/installer"
)
-func SetupInfra(env Env, st *state) []Task {
+func SetupInfra(env Env, startIP net.IP, st *state) []Task {
t := newLeafTask("Create client", func() error {
repo, err := st.ssClient.GetRepo("config")
if err != nil {
@@ -31,10 +32,10 @@
&t,
newConcurrentParentTask(
"Core services",
- SetupNetwork(env, st),
+ SetupNetwork(env, startIP, st),
SetupCertificateIssuers(env, st),
SetupAuth(env, st),
- SetupHeadscale(env, st),
+ SetupHeadscale(env, startIP, st),
SetupWelcome(env, st),
SetupAppStore(env, st),
),
@@ -101,13 +102,31 @@
return &t
}
-func SetupNetwork(env Env, st *state) Task {
+func SetupNetwork(env Env, startIP net.IP, st *state) Task {
t := newLeafTask("Setup network", func() error {
- ingressPrivateIP, err := netip.ParseAddr("10.1.0.1")
+ startAddr, err := netip.ParseAddr(startIP.String())
if err != nil {
return err
}
+ if !startAddr.Is4() {
+ return fmt.Errorf("Expected IPv4, got %s instead", startAddr)
+ }
+ addr := startAddr.AsSlice()
+ if addr[3] != 0 {
+ return fmt.Errorf("Expected last byte to be zero, got %d instead", addr[3])
+ }
+ addr[3] = 10
+ fromIP, ok := netip.AddrFromSlice(addr)
+ if !ok {
+ return fmt.Errorf("Must not reach")
+ }
+ addr[3] = 254
+ toIP, ok := netip.AddrFromSlice(addr)
+ if !ok {
+ return fmt.Errorf("Must not reach")
+ }
{
+ ingressPrivateIP := startAddr
headscaleIP := ingressPrivateIP.Next()
app, err := st.appsRepo.Find("metallb-ipaddresspool")
if err != nil {
@@ -133,8 +152,8 @@
}
if err := st.appManager.Install(app, st.nsGen, st.emptySuffixGen, map[string]any{
"name": env.Name,
- "from": "10.1.0.100", // TODO(gio): auto-generate
- "to": "10.1.0.254",
+ "from": fromIP.String(),
+ "to": toIP.String(),
"autoAssign": false,
"namespace": "metallb-system",
}); err != nil {
@@ -150,7 +169,7 @@
"privateNetwork": map[string]any{
"hostname": "private-network-proxy",
"username": "private-network-proxy",
- "ipSubnet": "10.1.0.0/24",
+ "ipSubnet": fmt.Sprintf("%s/24", startIP.String()),
},
}); err != nil {
return err
@@ -210,7 +229,7 @@
)
}
-func SetupHeadscale(env Env, st *state) Task {
+func SetupHeadscale(env Env, startIP net.IP, st *state) Task {
t := newLeafTask("Setup", func() error {
app, err := st.appsRepo.Find("headscale")
if err != nil {
@@ -218,6 +237,7 @@
}
if err := st.appManager.Install(app, st.nsGen, st.emptySuffixGen, map[string]any{
"subdomain": "headscale",
+ "ipSubnet": fmt.Sprintf("%s/24", startIP),
}); err != nil {
return err
}
diff --git a/core/installer/values-tmpl/headscale.cue b/core/installer/values-tmpl/headscale.cue
index 5cfec3c..e3453ba 100644
--- a/core/installer/values-tmpl/headscale.cue
+++ b/core/installer/values-tmpl/headscale.cue
@@ -1,5 +1,6 @@
input: {
subdomain: string
+ ipSubnet: string
}
name: "headscale"
@@ -58,7 +59,7 @@
ipAddressPool: "\(global.id)-headscale"
api: {
port: 8585
- rootDomain: global.domain
+ ipSubnet: input.ipSubnet
image: {
repository: images.api.fullName
tag: images.api.tag
diff --git a/core/installer/welcome/env.go b/core/installer/welcome/env.go
index 10a2ade..5436702 100644
--- a/core/installer/welcome/env.go
+++ b/core/installer/welcome/env.go
@@ -11,6 +11,7 @@
"log"
"net"
"net/http"
+ "net/netip"
"strings"
"github.com/gorilla/mux"
@@ -293,6 +294,10 @@
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
+ if err := s.repo.CommitAndPush(fmt.Sprintf("Allocate CIDR for %s", req.Name)); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
// if err := s.acceptInvitation(req.SecretToken); err != nil {
// http.Error(w, err.Error(), http.StatusInternalServerError)
// return
@@ -303,6 +308,27 @@
} else {
req.Name = name
}
+ var cidrs installer.EnvCIDRs
+ cidrsR, err := s.repo.Reader("env-cidrs.yaml")
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ defer cidrsR.Close()
+ if err := installer.ReadYaml(cidrsR, &cidrs); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ startIP, err := findNextStartIP(cidrs)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ cidrs = append(cidrs, installer.EnvCIDR{req.Name, startIP})
+ if err := s.repo.WriteYaml("env-cidrs.yaml", cidrs); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
t, dns := tasks.NewCreateEnvTask(
tasks.Env{
PCloudEnvName: env.Name,
@@ -315,6 +341,7 @@
net.ParseIP("135.181.48.180"),
net.ParseIP("65.108.39.172"),
},
+ startIP,
s.nsCreator,
s.repo,
)
@@ -331,3 +358,33 @@
go t.Start()
http.Redirect(w, r, fmt.Sprintf("/env/%s", key), http.StatusSeeOther)
}
+
+func findNextStartIP(cidrs installer.EnvCIDRs) (net.IP, error) {
+ m, err := netip.ParseAddr("10.0.0.0")
+ if err != nil {
+ return nil, err
+ }
+ for _, cidr := range cidrs {
+ i, err := netip.ParseAddr(cidr.IP.String())
+ if err != nil {
+ return nil, err
+ }
+ if i.Compare(m) > 0 {
+ m = i
+ }
+ }
+ sl := m.AsSlice()
+ sl[2]++
+ if sl[2] == 0b11111111 {
+ sl[2] = 0
+ sl[1]++
+ }
+ if sl[1] == 0b11111111 {
+ return nil, fmt.Errorf("Can not allocate")
+ }
+ ret, ok := netip.AddrFromSlice(sl)
+ if !ok {
+ return nil, fmt.Errorf("Must not reach")
+ }
+ return net.ParseIP(ret.String()), nil
+}