ClusterManager: Implements support of remote clusters.

After this change users will be able to:
* Create cluster and add/remove servers to it
* Install apps on remote cluster
* Move already installed apps between clusters
* Apps running on server being removed will auto-migrate
  to another server from that same cluster

This is achieved by:
* Installing and running minimal version of dodo on remote cluster
* Ingress-nginx is installed automatically on new clusters
* Next to nginx we run VPN client in the same pod, so that
  default cluster can establish secure communication with it
* Multiple reverse proxies are configured to get to the
  remote cluster service from ingress installed on default cluster.

Next steps:
* Support remote clusters in dodo apps (prototype ready)
* Clean up old cluster when moving app to the new one. Currently
  old cluster keeps running app pods even though no ingress can
  reach it anymore.

Change-Id: Iffc908c93416d4126a8e1c2832eae7b659cb8044
diff --git a/core/dns-api/Makefile b/core/dns-api/Makefile
index 1f127cb..bd78618 100644
--- a/core/dns-api/Makefile
+++ b/core/dns-api/Makefile
@@ -1,7 +1,7 @@
 repo_name ?= dtabidze
 podman ?= docker
 ifeq ($(podman), podman)
-manifest_dest=docker://docker.io/$(repo_name)/pcloud-installer:latest
+manifest_dest=docker://docker.io/$(repo_name)/dns-api:latest
 endif
 
 clean:
diff --git a/core/dns-api/main.go b/core/dns-api/main.go
index 7ab1bb2..c3d429d 100644
--- a/core/dns-api/main.go
+++ b/core/dns-api/main.go
@@ -23,6 +23,9 @@
 	if err != nil {
 		panic(err)
 	}
+	if err := store.Log(); err != nil {
+		panic(err)
+	}
 	server := NewServer(*port, *zone, ds, store, nameserverIP)
 	server.Start()
 }
diff --git a/core/dns-api/records_file.go b/core/dns-api/records_file.go
index 6916a5e..a25c342 100644
--- a/core/dns-api/records_file.go
+++ b/core/dns-api/records_file.go
@@ -35,11 +35,8 @@
 func (z *RecordsFile) DeleteTxtRecord(name, value string) {
 	z.lock.Lock()
 	defer z.lock.Unlock()
-	fmt.Printf("%s %s\n", name, value)
 	for i, rr := range z.rrs {
-		fmt.Printf("%+v\n", rr)
 		if txt, ok := rr.(*dns.TXT); ok {
-			fmt.Printf("%+v\n", txt)
 			if txt.Hdr.Name == name && strings.Join(txt.Txt, "") == value {
 				z.rrs = append(z.rrs[:i], z.rrs[i+1:]...)
 			}
@@ -47,6 +44,20 @@
 	}
 }
 
+func (z *RecordsFile) DeleteARecord(name, value string) error {
+	z.lock.Lock()
+	defer z.lock.Unlock()
+	for i, rr := range z.rrs {
+		if a, ok := rr.(*dns.A); ok {
+			if a.Hdr.Name == name && a.A.String() == value {
+				z.rrs = append(z.rrs[:i], z.rrs[i+1:]...)
+				return nil
+			}
+		}
+	}
+	return fmt.Errorf("not found")
+}
+
 // func (z *RecordsFile) DeleteRecordsFor(name string) {
 // 	z.lock.Lock()
 // 	defer z.lock.Unlock()
diff --git a/core/dns-api/server.go b/core/dns-api/server.go
index 37db8ff..d67ccf6 100644
--- a/core/dns-api/server.go
+++ b/core/dns-api/server.go
@@ -31,6 +31,8 @@
 	}
 	m.HandleFunc("/records-to-publish", s.recordsToPublish)
 	m.HandleFunc("/create-txt-record", s.createTxtRecord)
+	m.HandleFunc("/create-a-record", s.createARecord)
+	m.HandleFunc("/delete-a-record", s.deleteARecord)
 	m.HandleFunc("/delete-txt-record", s.deleteTxtRecord)
 	return s
 }
@@ -69,22 +71,44 @@
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
-	fmt.Printf("CREATE: %+v\n", req)
 	if err := s.store.Add(req.Entry, req.Text); err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 }
 
+func (s *Server) createARecord(w http.ResponseWriter, r *http.Request) {
+	var req record
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	if err := s.store.AddARecord(req.Entry, req.Text); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
 func (s *Server) deleteTxtRecord(w http.ResponseWriter, r *http.Request) {
 	var req record
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
-	fmt.Printf("DELETE: %+v\n", req)
 	if err := s.store.Delete(req.Entry, req.Text); err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 }
+
+func (s *Server) deleteARecord(w http.ResponseWriter, r *http.Request) {
+	var req record
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	if err := s.store.DeleteARecord(req.Entry, req.Text); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
diff --git a/core/dns-api/store.go b/core/dns-api/store.go
index 6d0a7c1..8dfd914 100644
--- a/core/dns-api/store.go
+++ b/core/dns-api/store.go
@@ -2,11 +2,16 @@
 
 import (
 	"fmt"
+	"io"
+	"os"
 )
 
 type RecordStore interface {
+	Log() error
 	Add(entry, txt string) error
+	AddARecord(entry, ip string) error
 	Delete(entry, txt string) error
+	DeleteARecord(entry, ip string) error
 }
 
 type fsRecordStore struct {
@@ -16,21 +21,32 @@
 	db       string
 }
 
+func (s *fsRecordStore) Log() error {
+	r, err := s.fs.Reader(s.db)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+	_, err = io.Copy(os.Stdout, r)
+	return err
+}
+
 func (s *fsRecordStore) read() (*RecordsFile, error) {
 	r, err := s.fs.Reader(s.db)
 	if err != nil {
 		return nil, err
 	}
 	defer r.Close()
-	return NewRecordsFile(r)
+	return NewRecordsFile(io.TeeReader(r, os.Stdout))
 }
+
 func (s *fsRecordStore) write(z *RecordsFile) error {
 	w, err := s.fs.Writer(s.db)
 	if err != nil {
 		return err
 	}
 	defer w.Close()
-	return z.Write(w)
+	return z.Write(io.MultiWriter(w, os.Stdout))
 }
 
 func (s *fsRecordStore) Add(entry, txt string) error {
@@ -46,6 +62,19 @@
 	return s.write(z)
 }
 
+func (s *fsRecordStore) AddARecord(entry, ip string) error {
+	z, err := s.read()
+	if err != nil {
+		return err
+	}
+	fqdn := fmt.Sprintf("%s.%s.", entry, s.zone)
+	z.CreateARecord(fqdn, ip)
+	// for _, ip := range s.publicIP {
+	// 	z.CreateARecord(fqdn, ip)
+	// }
+	return s.write(z)
+}
+
 func (s *fsRecordStore) Delete(entry, txt string) error {
 	z, err := s.read()
 	if err != nil {
@@ -56,3 +85,16 @@
 	// z.DeleteRecordsFor(fqdn)
 	return s.write(z)
 }
+
+func (s *fsRecordStore) DeleteARecord(entry, ip string) error {
+	z, err := s.read()
+	if err != nil {
+		return err
+	}
+	fqdn := fmt.Sprintf("%s.%s.", entry, s.zone)
+	if err := z.DeleteARecord(fqdn, ip); err != nil {
+		return err
+	}
+	// z.DeleteRecordsFor(fqdn)
+	return s.write(z)
+}