4 Commits

Author SHA1 Message Date
Bastian
4bfd79e79e improved: retrieve source/destination cluster name for better insights 2023-07-13 15:18:24 +02:00
Bastian
6e8eb7ce2b fixed: preflight checks 2023-07-13 14:45:55 +02:00
Bastian
be88cb4d40 fixed: perf_vm_stopped++ never counted. 2023-07-13 13:54:20 +02:00
Bastian
1343dc6b51 fixed: Correct target host now displayed in log messsage, Add downtime counter 2023-07-13 13:51:58 +02:00

View File

@@ -1,5 +1,8 @@
#!/bin/bash #!/bin/bash
# Cross Pool Migration and incremental replication Tool for Proxmox VMs using Ceph.
# Author: Bastian Mäuser <bma@netz.org>
LC_ALL="en_US.UTF-8" LC_ALL="en_US.UTF-8"
source rainbow.sh source rainbow.sh
@@ -13,14 +16,11 @@ declare opt_influx_jobname=''
declare opt_influx_job_metrics='crossover_xmit' declare opt_influx_job_metrics='crossover_xmit'
declare opt_influx_summary_metrics='crossover_jobs' declare opt_influx_summary_metrics='crossover_jobs'
# Cross Pool Migration and incremental replication Tool for Proxmox VMs using Ceph.
# Author: Bastian Mäuser <bma@netz.org>
name=$(basename "$0") name=$(basename "$0")
# readonly variables
declare -r NAME=$name declare -r NAME=$name
declare -r VERSION=0.8 declare -r VERSION=0.8
declare -r PROGNAME=${NAME%.*} declare -r PROGNAME=${NAME%.*}
declare -r PVE_DIR="/etc/pve" declare -r PVE_DIR="/etc/pve"
declare -r PVE_NODES="$PVE_DIR/nodes" declare -r PVE_NODES="$PVE_DIR/nodes"
declare -r QEMU='qemu-server' declare -r QEMU='qemu-server'
@@ -28,15 +28,19 @@ declare -r QEMU_CONF_CLUSTER="$PVE_NODES/*/$QEMU"
declare -r EXT_CONF='.conf' declare -r EXT_CONF='.conf'
declare -r PVFORMAT_FULL='e:%t r:%e c:%r a:%a %b %p' declare -r PVFORMAT_FULL='e:%t r:%e c:%r a:%a %b %p'
declare -r PVFORMAT_SNAP='e:%t c:%r a:%a %b' declare -r PVFORMAT_SNAP='e:%t c:%r a:%a %b'
logfile=$(mktemp) logfile=$(mktemp)
declare -r LOG_FILE=$logfile declare -r LOG_FILE=$logfile
# associative global arrays
declare -A -g pvnode declare -A -g pvnode
declare -A -g dstpvnode declare -A -g dstpvnode
declare -A -g svmids declare -A -g svmids
declare -A -g dvmids declare -A -g dvmids
declare -g scluster
declare -g dcluster
# global integers
declare -g -i perf_freeze_ok=0 declare -g -i perf_freeze_ok=0
declare -g -i perf_freeze_failed=0 declare -g -i perf_freeze_failed=0
declare -g -i perf_ss_ok=0 declare -g -i perf_ss_ok=0
@@ -56,6 +60,7 @@ declare -g -i perf_snaps_removed=0
declare -g -i perf_vm_total=0 declare -g -i perf_vm_total=0
declare -g -i perf_vm_ok=0 declare -g -i perf_vm_ok=0
# commandline parameters
declare opt_destination declare opt_destination
declare opt_vm_ids='' declare opt_vm_ids=''
declare opt_snapshot_prefix='mirror-' declare opt_snapshot_prefix='mirror-'
@@ -309,20 +314,6 @@ function get_vm_ids(){
echo "$data" echo "$data"
} }
function get_config_file(){
local file_config=''
if exist_file "$QEMU_CONF_CLUSTER/$vm_id$EXT_CONF"; then
file_config=$(ls $QEMU_CONF_CLUSTER/$vm_id$EXT_CONF)
else
log error "VM $vm_id - Unknown technology or VMID not found: $QEMU_CONF_CLUSTER/$vm_id$EXT_CONF"
end_process 128
fi
echo "$file_config"
}
function get_disks_from_config(){ function get_disks_from_config(){
local disks; local disks;
local file_config=$1 local file_config=$1
@@ -415,6 +406,8 @@ function mirror() {
local -i endjob local -i endjob
local -i vmcount=0 local -i vmcount=0
local -i diskcount=0 local -i diskcount=0
local -i startdowntime
local -i enddowntime
local disp_perf_freeze_failed local disp_perf_freeze_failed
local disp_perf_ss_failed local disp_perf_ss_failed
@@ -439,6 +432,9 @@ function mirror() {
end_process 1 end_process 1
fi fi
scluster=$(grep cluster_name /etc/pve/corosync.conf | cut -d " " -f 4)
dcluster=$(ssh "$opt_destination" grep cluster_name /etc/pve/corosync.conf | cut -d " " -f 4)
map_source_to_destination_vmid map_source_to_destination_vmid
map_vmids_to_host map_vmids_to_host
map_vmids_to_dsthost "$opt_destination" map_vmids_to_dsthost "$opt_destination"
@@ -449,9 +445,12 @@ function mirror() {
fi fi
for vm_id in $svmids; do for vm_id in $svmids; do
file_config="$PVE_NODES/${pvnode[$vm_id]}/$QEMU/$vm_id.conf"
if ! exist_file "$file_config"; then
log error "VM $vm_id - Preflight check: VM $vm_id does not exist on source cluster [$scluster] - skipping to next VM."
continue
fi
(( vmcount++ )) (( vmcount++ ))
local file_config; file_config=$(get_config_file)
[ -z "$file_config" ] && continue
local disk='' local disk=''
dvmid=${dvmids[$vm_id]} dvmid=${dvmids[$vm_id]}
vmname=$(cat $PVE_NODES/"${pvnode[$vm_id]}"/$QEMU/"$vm_id".conf | sed -e ''$restripsnapshots'' | grep "name\:" | cut -d' ' -f 2) vmname=$(cat $PVE_NODES/"${pvnode[$vm_id]}"/$QEMU/"$vm_id".conf | sed -e ''$restripsnapshots'' | grep "name\:" | cut -d' ' -f 2)
@@ -467,7 +466,7 @@ function mirror() {
if [ $host_on_destination ]; then if [ $host_on_destination ]; then
dststatus=$(ssh root@${dstpvnode[$dvmid]} qm status $dvmid|cut -d' ' -f 2) dststatus=$(ssh root@${dstpvnode[$dvmid]} qm status $dvmid|cut -d' ' -f 2)
if [ $dststatus == "running" ]; then if [ $dststatus == "running" ]; then
log error "VM is running on Destination. bailing out" log error "VM is running on Destination Cluster [$dcluster]. bailing out"
end_process 255 end_process 255
fi fi
fi fi
@@ -496,6 +495,7 @@ function mirror() {
do_run "ha-manager remove $vm_id" do_run "ha-manager remove $vm_id"
fi fi
do_run "ssh root@${pvnode[$vm_id]} qm shutdown $vm_id >/dev/null" do_run "ssh root@${pvnode[$vm_id]} qm shutdown $vm_id >/dev/null"
startdowntime=$(date +%s)
fi fi
#Lock on source + destination #Lock on source + destination
@@ -638,8 +638,10 @@ function mirror() {
fi fi
#--migrate so start on destination? #--migrate so start on destination?
if [ $opt_migrate -eq 1 ]; then if [ $opt_migrate -eq 1 ]; then
log info "VM $dvmid - Starting VM on ${pvnode[$vm_id]}" log info "VM $dvmid - Starting VM on node ${dstpvnode[$dvmid]} in cluster [$dcluster]"
do_run "ssh root@""${dstpvnode[$dvmid]}"" qm start "$dvmid >/dev/null do_run "ssh root@""${dstpvnode[$dvmid]}"" qm start "$dvmid >/dev/null
enddowntime=$(date +%s)
log info "VM $dvmid - Downtime: $(( enddowntime - startdowntime )) Seconds"
fi fi
done done
@@ -722,8 +724,8 @@ function vm_freeze() {
status=$(ssh root@"$fhost" qm status "$fvm"|cut -d' ' -f 2) status=$(ssh root@"$fhost" qm status "$fvm"|cut -d' ' -f 2)
if ! [[ "$status" == "running" ]]; then if ! [[ "$status" == "running" ]]; then
log info "VM $fvm - Not running, skipping fsfreeze-freeze" log info "VM $fvm - Not running, skipping fsfreeze-freeze"
return
(( perf_vm_stopped++ )) (( perf_vm_stopped++ ))
return
else else
(( perf_vm_running++ )) (( perf_vm_running++ ))
fi fi
@@ -798,7 +800,7 @@ function end_process(){
local -i bps local -i bps
local -i ss_total local -i ss_total
local subject local subject
if ! [[ -z "$startjob" && -z "$endjob" ]]; then if ! [[ -z "$startjob" || -z "$endjob" ]]; then
runtime=$(expr $endjob - $startjob) runtime=$(expr $endjob - $startjob)
bps=$(expr $perf_bytes_total/$runtime) bps=$(expr $perf_bytes_total/$runtime)
fi fi