ClusterManager: Implements support of remote clusters.
After this change users will be able to:
* Create cluster and add/remove servers to it
* Install apps on remote cluster
* Move already installed apps between clusters
* Apps running on server being removed will auto-migrate
to another server from that same cluster
This is achieved by:
* Installing and running minimal version of dodo on remote cluster
* Ingress-nginx is installed automatically on new clusters
* Next to nginx we run VPN client in the same pod, so that
default cluster can establish secure communication with it
* Multiple reverse proxies are configured to get to the
remote cluster service from ingress installed on default cluster.
Next steps:
* Support remote clusters in dodo apps (prototype ready)
* Clean up old cluster when moving app to the new one. Currently
old cluster keeps running app pods even though no ingress can
reach it anymore.
Change-Id: Iffc908c93416d4126a8e1c2832eae7b659cb8044
diff --git a/core/installer/values-tmpl/appmanager.cue b/core/installer/values-tmpl/appmanager.cue
index 24d0d8d..1cd7dbf 100644
--- a/core/installer/values-tmpl/appmanager.cue
+++ b/core/installer/values-tmpl/appmanager.cue
@@ -69,7 +69,10 @@
values: {
repoAddr: input.repoAddr
sshPrivateKey: base64.Encode(null, input.sshPrivateKey)
+ // TODO(gio): de-hardcode these variables
headscaleAPIAddr: "http://headscale-api.\(global.namespacePrefix)app-headscale.svc.cluster.local"
+ dnsAPIAddr: "http://dns-api.\(global.namespacePrefix)dns.svc.cluster.local"
+ clusterProxyConfigPath: "/apps/private-network/resources/proxy-backend-config.yaml"
ingress: {
className: input.network.ingressClass
domain: _domain
diff --git a/core/installer/values-tmpl/certificate-issuer-custom.cue b/core/installer/values-tmpl/certificate-issuer-custom.cue
index 8721e7f..469382d 100644
--- a/core/installer/values-tmpl/certificate-issuer-custom.cue
+++ b/core/installer/values-tmpl/certificate-issuer-custom.cue
@@ -47,6 +47,7 @@
helm: {
"certificate-issuer": {
chart: charts["certificate-issuer"]
+ Info: "Configuring SSL certificate issuer for \(input.domain)"
dependsOn: [{
name: "ingress-nginx"
namespace: "\(global.namespacePrefix)ingress-private"
diff --git a/core/installer/values-tmpl/cluster-network.cue b/core/installer/values-tmpl/cluster-network.cue
new file mode 100644
index 0000000..d470ff1
--- /dev/null
+++ b/core/installer/values-tmpl/cluster-network.cue
@@ -0,0 +1,138 @@
+import (
+ // "encoding/base64"
+)
+
+input: {
+ cluster: #Cluster
+ vpnUser: string
+ vpnProxyHostname: string
+ vpnAuthKey: string @role(VPNAuthKey) @usernameField(vpnUser)
+ // TODO(gio): support port allocator
+}
+
+name: "Cluster Network"
+namespace: "cluster-network"
+
+out: {
+ images: {
+ "ingress-nginx": {
+ registry: "registry.k8s.io"
+ repository: "ingress-nginx"
+ name: "controller"
+ tag: "v1.8.0"
+ pullPolicy: "IfNotPresent"
+ }
+ "tailscale-proxy": {
+ repository: "tailscale"
+ name: "tailscale"
+ tag: "v1.42.0"
+ pullPolicy: "IfNotPresent"
+ }
+ // portAllocator: {
+ // repository: "giolekva"
+ // name: "port-allocator"
+ // tag: "latest"
+ // pullPolicy: "Always"
+ // }
+ }
+
+ charts: {
+ "access-secrets": {
+ kind: "GitRepository"
+ address: "https://code.v1.dodo.cloud/helm-charts"
+ branch: "main"
+ path: "charts/access-secrets"
+ }
+ "ingress-nginx": {
+ kind: "GitRepository"
+ address: "https://code.v1.dodo.cloud/helm-charts"
+ branch: "main"
+ path: "charts/ingress-nginx"
+ }
+ "tailscale-proxy": {
+ kind: "GitRepository"
+ address: "https://code.v1.dodo.cloud/helm-charts"
+ branch: "main"
+ path: "charts/tailscale-proxy"
+ }
+ // portAllocator: {
+ // kind: "GitRepository"
+ // address: "https://code.v1.dodo.cloud/helm-charts"
+ // branch: "main"
+ // path: "charts/port-allocator"
+ // }
+ }
+
+ helm: {
+ _fullnameOverride: "\(global.id)-nginx-cluster-\(input.cluster.name)"
+ "access-secrets": {
+ chart: charts["access-secrets"]
+ values: {
+ serviceAccountName: _fullnameOverride
+ }
+ }
+ "ingress-nginx": {
+ chart: charts["ingress-nginx"]
+ dependsOn: [{
+ name: "access-secrets"
+ namespace: release.namespace
+ }]
+ values: {
+ fullnameOverride: _fullnameOverride
+ controller: {
+ service: enabled: false
+ ingressClassByName: true
+ ingressClassResource: {
+ name: input.cluster.ingressClassName
+ enabled: true
+ default: false
+ controllerValue: "k8s.io/\(input.cluster.name)"
+ }
+ config: {
+ "proxy-body-size": "200M" // TODO(giolekva): configurable
+ "force-ssl-redirect": "true"
+ "server-snippet": """
+ more_clear_headers "X-Frame-Options";
+ """
+ }
+ admissionWebhooks: {
+ enabled: false
+ }
+ image: {
+ registry: images["ingress-nginx"].registry
+ image: images["ingress-nginx"].imageName
+ tag: images["ingress-nginx"].tag
+ pullPolicy: images["ingress-nginx"].pullPolicy
+ }
+ extraContainers: [{
+ name: "proxy"
+ image: images["tailscale-proxy"].fullNameWithTag
+ env: [{
+ name: "TS_AUTHKEY"
+ value: input.vpnAuthKey
+ }, {
+ name: "TS_HOSTNAME"
+ value: input.vpnProxyHostname
+ }, {
+ name: "TS_EXTRA_ARGS"
+ value: "--login-server=https://headscale.\(global.domain)"
+ }]
+ }]
+ }
+ }
+ }
+ // "port-allocator": {
+ // chart: charts.portAllocator
+ // values: {
+ // repoAddr: release.repoAddr
+ // sshPrivateKey: base64.Encode(null, input.sshPrivateKey)
+ // ingressNginxPath: "\(release.appDir)/resources/ingress-nginx.yaml"
+ // image: {
+ // repository: images.portAllocator.fullName
+ // tag: images.portAllocator.tag
+ // pullPolicy: images.portAllocator.pullPolicy
+ // }
+ // }
+ // }
+ }
+}
diff --git a/core/installer/values-tmpl/virtual-machine.cue b/core/installer/values-tmpl/virtual-machine.cue
index 6f311d9..841cb8b 100644
--- a/core/installer/values-tmpl/virtual-machine.cue
+++ b/core/installer/values-tmpl/virtual-machine.cue
@@ -4,7 +4,8 @@
authKey?: string @name(Auth Key) @role(VPNAuthKey) @usernameField(username) @enabledField(vpnEnabled)
cpuCores: int | *1 @name(CPU Cores)
memory: string | *"2Gi" @name(Memory)
- vpnEnabled: bool @name(Enable VPN)
+ vpnEnabled?: bool @name(Enable VPN)
+ codeServerEnabled?: bool @name(Install VSCode Server)
}
name: "Virutal Machine"
@@ -21,15 +22,20 @@
domain: global.domain
cpuCores: input.cpuCores
memory: input.memory
- if !input.vpnEnabled {
- vpn: enabled: false
- }
- if input.vpnEnabled {
- vpn: {
- enabled: true
- loginServer: "https://headscale.\(global.domain)"
- authKey: input.authKey
+ if input.vpnEnabled != _|_ {
+ if !input.vpnEnabled {
+ vpn: enabled: false
}
+ if input.vpnEnabled {
+ vpn: {
+ enabled: true
+ loginServer: "https://headscale.\(global.domain)"
+ authKey: input.authKey
+ }
+ }
+ }
+ if input.codeServerEnabled != _|_ {
+ codeServerEnabled: input.codeServerEnabled
}
}
}