Ensuring Zero Downtime: On-Prem High Availability Docker Swarm Setup with Proxmox

#!/bin/bash
# ====== CONFIGURATION ======
MANAGERS=("10.1.1.61" "10.1.1.62" "10.1.1.60")
MANAGER_USER="xxxxxx"
MANAGER_PASS="xxxxxx"

# Proxmox
PROXMOX_HOST="10.1.1.35"
PROXMOX_USER="xxxxxx"
PROXMOX_PASS="xxxxxx"
PROXMOX_VM_ID=109

# New Node (standby VM details)
NEW_NODE_IP="10.1.1.63"
NEW_NODE_HOSTNAME="new-manager-node"
NEW_NODE_USER="xxxxxx"
NEW_NODE_PASS="xxxxxx"
SWARM_JOIN_CMD="docker swarm join --token SWMTKN-1-xxxxx-xxxxx 10.1.1.62:2377"

# ====== FUNCTION TO CHECK IF A NODE IS UP ======
check_node_up() {
  curl -s --connect-timeout 3 http://$1:9100/metrics > /dev/null
  return $?
}

# ====== CHECK ALL MANAGERS ======
echo " Checking Swarm manager nodes..."
any_down=false
for ip in "${MANAGERS[@]}"; do
  if check_node_up "$ip"; then
    echo "✅ Manager $ip is UP"
  else
    echo "❌ Manager $ip is DOWN"
    any_down=true
  fi
done

# ====== IF ALL MANAGERS ARE UP, EXIT ======
if ! $any_down; then
  echo " All managers are healthy. No action needed."
  exit 0
fi

# ====== START STANDBY VM FROM PROXMOX ======
echo " Starting VM $PROXMOX_VM_ID via Proxmox ($PROXMOX_HOST)..."
sshpass -p "$PROXMOX_PASS" ssh -o StrictHostKeyChecking=no ${PROXMOX_USER}@${PROXMOX_HOST} "qm start ${PROXMOX_VM_ID}"

echo " Waiting for ${NEW_NODE_IP} to be reachable via SSH..."
until nc -z ${NEW_NODE_IP} 22; do
  echo "   ... still waiting ..."
  sleep 5
done

echo " VM ${NEW_NODE_IP} is online."
echo " Waiting 30 seconds for the new VM ($NEW_NODE_IP) to boot..."
sleep 30

# ====== JOIN THE NEW NODE TO THE SWARM ======
sshpass -p "$NEW_NODE_PASS" ssh -tt -o StrictHostKeyChecking=no ${NEW_NODE_USER}@${NEW_NODE_IP} "echo $NEW_NODE_PASS | sudo -S ${SWARM_JOIN_CMD}"

# ====== PROMOTE THE NEW NODE TO MANAGER ======
for manager_ip in "${MANAGERS[@]}"; do
  if check_node_up "$manager_ip"; then
    echo " Promoting $NEW_NODE_HOSTNAME to manager from $manager_ip..."
    sshpass -p "$MANAGER_PASS" ssh -o StrictHostKeyChecking=no ${MANAGER_USER}@${manager_ip} "echo $MANAGER_PASS | sudo -S docker node promote $NEW_NODE_HOSTNAME"
    break
  fi
done

echo " Swarm repair completed."