monitoring: nodes reboots itself if other nodes can not reach it
diff --git a/scripts/homelab/authorized_keys b/scripts/homelab/authorized_keys
new file mode 100644
index 0000000..6f9d19c
--- /dev/null
+++ b/scripts/homelab/authorized_keys
@@ -0,0 +1,5 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICt/rv/rpYQwqZWDyeaawvkDVQSYlciMQ7h0F7YlYu+p pcloud@rpi111
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOWBTWdkCzE/wwGvDl4rNHlwnboswgSTjmOsfeqdHfgl pcloud@rpi112
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAUwOoo9JoV7rdSamjIPQUu1XWwOjnmbj8e4OMnRTNc9 pcloud@rpi113
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAWpwC256L5A5XzL6IqO3mHpHCJrFyjyP/gt/UhrUH4c pcloud@rpi114
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBbMNLwf7yGluwSrUv2YcAbBqFQW6fBgGAS7HJLcqfRI pcloud@rpi116
diff --git a/scripts/homelab/check-ssh-login-cron b/scripts/homelab/check-ssh-login-cron
index de7f8bb..b184ac9 100644
--- a/scripts/homelab/check-ssh-login-cron
+++ b/scripts/homelab/check-ssh-login-cron
@@ -1 +1 @@
-*/5 * * * * pi /home/pi/homelab/check-ssh-login.sh
+*/10 * * * * pcloud /home/pcloud/pcloud-tools/check-ssh-login.sh
diff --git a/scripts/homelab/check-ssh-login.sh b/scripts/homelab/check-ssh-login.sh
index f8241df..5c3a4a6 100755
--- a/scripts/homelab/check-ssh-login.sh
+++ b/scripts/homelab/check-ssh-login.sh
@@ -1,5 +1,16 @@
-#!/bin/sh
+#!/bin/bash
-ssh pcloud@192.168.0.111 "touch ~/SSH_LOGGED_IN"
-ssh pcloud@192.168.0.112 "touch ~/SSH_LOGGED_IN"
-ssh pcloud@192.168.0.113 "touch ~/SSH_LOGGED_IN"
+MY_IPS=( $(ifconfig | grep -Eo "([0-9]*\.){3}[0-9]*") )
+echo $MY_IPS
+SERVERS=(192.168.0.111 192.168.0.112 192.168.0.113 192.168.0.114 192.168.0.116)
+
+sleep $[ ( $RANDOM % 180 ) + 1 ]
+
+for IP in "${SERVERS[@]}"
+do
+ if [[ ! "${MY_IPS[*]}" =~ "$IP" ]];
+ then
+ echo $IP
+ ssh -oStrictHostKeyChecking=no "pcloud@${IP}" "touch ~/SSH_LOGGED_IN"
+ fi
+done
diff --git a/scripts/homelab/copy-pcloud-tools.sh b/scripts/homelab/copy-pcloud-tools.sh
new file mode 100755
index 0000000..df79bed
--- /dev/null
+++ b/scripts/homelab/copy-pcloud-tools.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+SERVERS=(192.168.0.111 192.168.0.112 192.168.0.113 192.168.0.114 192.168.0.116)
+
+# source gather-pub-keys.sh
+
+for IP in "${SERVERS[@]}"
+do
+ echo $IP
+ # ssh "pcloud@${IP}" "rm -rf /home/pcloud/pcloud-tools"
+ # ssh "pcloud@${IP}" "mkdir /home/pcloud/pcloud-tools"
+ # scp authorized_keys "pcloud@${IP}:/home/pcloud/pcloud-tools"
+ # ssh "pcloud@${IP}" "cat /home/pcloud/pcloud-tools/authorized_keys >> /home/pcloud/.ssh/authorized_keys"
+ # scp zap-disk.sh "pcloud@${IP}:/home/pcloud/pcloud-tools"
+ # scp generate-ssh-key.sh "pcloud@${IP}:/home/pcloud/pcloud-tools"
+ scp check-ssh-login.sh "pcloud@${IP}:/home/pcloud/pcloud-tools"
+ scp restart-if-no-ssh-login.sh "pcloud@${IP}:/home/pcloud/pcloud-tools"
+ scp check-ssh-login-cron "pcloud@${IP}:/home/pcloud/pcloud-tools"
+ scp restart-if-no-ssh-login-cron "pcloud@${IP}:/home/pcloud/pcloud-tools"
+ ssh "pcloud@${IP}" "sudo mv /home/pcloud/pcloud-tools/check-ssh-login-cron /etc/cron.d/check-ssh-login"
+ ssh "pcloud@${IP}" "sudo mv /home/pcloud/pcloud-tools/restart-if-no-ssh-login-cron /etc/cron.d/restart-if-no-ssh-login"
+ ssh "pcloud@${IP}" "sudo chown root:root /etc/cron.d/check-ssh-login"
+ ssh "pcloud@${IP}" "sudo chown root:root /etc/cron.d/restart-if-no-ssh-login"
+done
diff --git a/scripts/homelab/gather-pub-keys.sh b/scripts/homelab/gather-pub-keys.sh
new file mode 100755
index 0000000..5c1ba3f
--- /dev/null
+++ b/scripts/homelab/gather-pub-keys.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+SERVERS=(192.168.0.111 192.168.0.112 192.168.0.113 192.168.0.114 192.168.0.116)
+
+rm -f authorized_keys
+touch authorized_keys
+
+for IP in "${SERVERS[@]}"
+do
+ ssh "pcloud@${IP}" "sh /home/pcloud/pcloud-tools/generate-ssh-key.sh"
+ scp "pcloud@${IP}:/home/pcloud/.ssh/id_ed25519.pub" tmp-key.pub
+ cat tmp-key.pub >> authorized_keys
+ rm tmp-key.pub
+done
+
diff --git a/scripts/homelab/generate-ssh-key.sh b/scripts/homelab/generate-ssh-key.sh
new file mode 100755
index 0000000..57ff65d
--- /dev/null
+++ b/scripts/homelab/generate-ssh-key.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -f /home/pcloud/.ssh/id_ed25519 ];
+then
+ exit 0
+fi
+
+ssh-keygen -t ed25519 -b 4096 -N "" -f /home/pcloud/.ssh/id_ed25519
diff --git a/scripts/homelab/restart-if-no-ssh-login-cron b/scripts/homelab/restart-if-no-ssh-login-cron
index 663d3b9..620e7dc 100644
--- a/scripts/homelab/restart-if-no-ssh-login-cron
+++ b/scripts/homelab/restart-if-no-ssh-login-cron
@@ -1 +1 @@
-*/13 * * * * pcloud /home/pcloud/pcloud-tools/restart-if-no-ssh-login.sh
+*/30 * * * * pcloud /home/pcloud/pcloud-tools/restart-if-no-ssh-login.sh
diff --git a/scripts/homelab/restart-if-no-ssh-login.sh b/scripts/homelab/restart-if-no-ssh-login.sh
index 39ddf6e..5527440 100755
--- a/scripts/homelab/restart-if-no-ssh-login.sh
+++ b/scripts/homelab/restart-if-no-ssh-login.sh
@@ -1,8 +1,8 @@
-if [ ! -f /home/pcloud/SSH_LOGGED_IN ];
+#!/bin/bash
+
+if [[ ! -f /home/pcloud/SSH_LOGGED_IN ]];
then
- echo "SSH_LOGGED_IN not found, restaring"
sudo shutdown -r
else
- echo "SSH_LOGGED_IN found and removing"
rm /home/pcloud/SSH_LOGGED_IN
fi
diff --git a/scripts/homelab/zap-disk.sh b/scripts/homelab/zap-disk.sh
new file mode 100755
index 0000000..2f24969
--- /dev/null
+++ b/scripts/homelab/zap-disk.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+DISK="/dev/sda3"
+
+# Zap the disk to a fresh, usable state (zap-all is important, b/c MBR has to be clean)
+
+# You will have to run this step for all disks.
+sgdisk --zap-all $DISK
+
+# Clean hdds with dd
+dd if=/dev/zero of="$DISK" bs=1M count=100 oflag=direct,dsync
+
+# Clean disks such as ssd with blkdiscard instead of dd
+blkdiscard $DISK
+
+# These steps only have to be run once on each node
+# If rook sets up osds using ceph-volume, teardown leaves some devices mapped that lock the disks.
+ls /dev/mapper/ceph-* | xargs -I% -- dmsetup remove %
+
+# ceph-volume setup can leave ceph-<UUID> directories in /dev and /dev/mapper (unnecessary clutter)
+rm -rf /dev/ceph-*
+rm -rf /dev/mapper/ceph--*
+
+# Inform the OS of partition table changes
+partprobe $DISK