diff --git a/crossover b/crossover index 198166d..beea20f 100755 --- a/crossover +++ b/crossover @@ -319,12 +319,22 @@ function mirror() { local timestamp; timestamp=$(date +%Y%m%d%H%M%S) local xmittype - local xmitrc - local freezerc - local unfreezerc + local -i xmitrc + local -i freezerc + local -i unfreezerc + local -i startdisk + local -i enddisk + local -i startjob + local -i endjob + local -i fullbytes + local -i incrementalbytes + local -i totalbytes + local -i vmcount=0 + local -i diskcount=0 log info "ACTION: Onlinemirror" log info "Start mirror $(date "+%F %T")" + startjob=$(date +%s) #create pid file local pid_file="/var/run/$PROGNAME.pid" @@ -345,6 +355,7 @@ function mirror() { map_vmids_to_dsthost "$opt_destination" for vm_id in $svmids; do + (( vmcount++ )) local file_config; file_config=$(get_config_file) [ -z "$file_config" ] && continue local disk='' @@ -384,30 +395,33 @@ function mirror() { #Lock on source + destination if [ $opt_lock -eq 1 ]; then - ssh root@"${pvnode[$vm_id]}" qm set "$vm_id" --lock backup - fi - - if [ $opt_lock -eq 1 ]; then - ssh root@"${dstpvnode[$dvmid]}" qm set "$dvmid" --lock backup + do_run "ssh root@"${pvnode[$vm_id]}" qm set "$vm_id" --lock backup" >/dev/null + log info "VM $vm_id - locked $vm_id [rc:$?]" + do_run "ssh root@"${dstpvnode[$dvmid]}" qm set "$dvmid" --lock backup" >/dev/null + log info "VM $dvmid - locked $dvmid [rc:$?]" fi vm_freeze "$vm_id" "${pvnode[$vm_id]}" >/dev/null freezerc=$? + if [ $freezerc -gt 0 ]; then + log error "VM $vm_id - QEMU-Guest could not fsfreeze on guest." + fi for disk in $(get_disks_from_config "$file_config"); do src_image_spec=$(get_image_spec "$disk") - create_snapshot "$src_image_spec@$opt_snapshot_prefix$timestamp" + create_snapshot "$src_image_spec@$opt_snapshot_prefix$timestamp" 2>/dev/null done vm_unfreeze "$vm_id" "${pvnode[$vm_id]}" >/dev/null unfreezerc=$? if [ $unfreezerc -gt 0 ]; then - log error "VM $vm_id - QEMU-Guest could not fsfreeze on guest." + log error "VM $vm_id - QEMU-Guest could not fsunfreeze on guest." fi if [ ! $opt_keepslock -eq 1 ]; then - ssh root@${pvnode[$vm_id]} qm unlock $vm_id - log info "VM $vm_id - Unlocking source VM $vm_id" + do_run "ssh root@${pvnode[$vm_id]} qm unlock $vm_id" >/dev/null + log info "VM $vm_id - unlocked source VM $vm_id [rc:$?]" fi for disk in $(get_disks_from_config "$file_config"); do + (( diskcount++ )) log debug "VMID: $vm_id Disk: $disk DESTVMID: $dvmid" src_image_spec=$(get_image_spec "$disk") [ -z "$src_image_spec" ] && continue @@ -446,7 +460,9 @@ function mirror() { xmitjob="rbd export --rbd-concurrent-management-ops 8 $src_image_pool/$src_image_name$snapshot_name --no-progress - | tee >({ wc -c; } >/tmp/$PROGNAME.$pid.$dst_image_pool-$dst_image_name.size) | pv -s $snapshotsize -F \"VM $vm_id - Full xmit: $PVFORMAT_FULL\" | ssh $opt_destination rbd import --image-format 2 - $dst_image_pool/$dst_image_name" # create initial snapshot on destination log debug "xmitjob: $xmitjob" + startdisk=$(date +%s) do_run "$xmitjob" + enddisk=$(date +%s) xmitrc=$? if [ ! $xmitrc ]; then log error "Transmitting Image failed" @@ -466,17 +482,21 @@ function mirror() { fi xmitjob="rbd export-diff --no-progress --from-snap $opt_snapshot_prefix$basets $src_image_pool/$currentlocal - | tee >({ wc -c; } >/tmp/$PROGNAME.$pid.$dst_image_pool-$dst_image_name.size) | pv -F \"VM $vm_id - Snap xmit: $PVFORMAT_SNAP\" | ssh $opt_destination rbd import-diff --no-progress - $dst_image_pool/$dst_image_name" log debug "xmitjob: $xmitjob" + startdisk=$(date +%s) do_run "$xmitjob" + enddisk=$(date +%s) xmitrc=$? if [ ! $xmitrc ]; then log error "Transmitting Image failed" return 1 fi - do_housekeeping "localhost" "$src_image_pool" "$src_image_name" $opt_keep_local - do_housekeeping "$opt_destination" "$dst_image_pool" "$dst_image_name" $opt_keep_remote + do_housekeeping "localhost" "$src_image_pool" "$src_image_name" "$opt_keep_local" "$vm_id" + do_housekeeping "$opt_destination" "$dst_image_pool" "$dst_image_name" "$opt_keep_remote" "$vm_id" fi xmitted=$(cat /tmp/"$PROGNAME"."$pid"."$dst_image_pool"-"$dst_image_name".size) + totalbytes=$(( totalbytes + xmitted )) rm /tmp/"$PROGNAME"."$pid"."$dst_image_pool"-"$dst_image_name".size + log info "VM $vm_id - Disk Summary: Took $(( enddisk - startdisk )) Seconds to transfer $xmitted Bytes in a $xmittype run" if [ -n "$opt_influx_api_url" ]; then log info "VM $vm_id - Logging to InfluxDB: $opt_influx_api_url" influxlp="$opt_influx_measurement,destination=$opt_destination,srcimage=$src_image_name,dstimage=$dst_image_name,xmittype=$xmittype bytescalculated=$snapshotsize""i,bytesonwire=$xmitted""i,xmitrc=$xmitrc""i,freezerc=$freezerc""i,unfreezerc=$unfreezerc""i,basets=$basets""i" @@ -492,7 +512,9 @@ function mirror() { log info "VM $dvmid - Unlocking destination VM $dvmid" fi done + endjob=$(date +%s) log info "Finnished mirror $(date "+%F %T")" + log info "Job Summary: Took $(( endjob - startjob )) Seconds to transfer $totalbytes Bytes for $diskcount Disks on $vmcount VMs" } function do_housekeeping(){ @@ -500,23 +522,25 @@ function do_housekeeping(){ rbdpool=$2 rbdimage=$3 keep=$4 + vm_id=$5 snapshotstokill=$(ssh $horst rbd ls -l $rbdpool | grep $rbdimage@$opt_snapshot_prefix | cut -d ' ' -f 1|head -n -1 |head -n -$keep) - log info "Houskeeping $horst $rbdpool $rbdimage, keeping previous $keep Snapshots" + #log info "VM $vm_id - Houskeeping $horst $rbdpool $rbdimage, keeping previous $keep Snapshots" for snap in $snapshotstokill; do cmd="ssh $horst rbd snap rm $rbdpool/$snap" - if ! do_run $cmd; then - log error "Housekeeping failed: $cmd" - return 1 - fi + do_run $cmd 2>/dev/null + log info "VM $vm_id - Housekeeping: $horst $rbdpool $rbdimage, keeping previous $keep Snapshots" + log debug "$cmd" + return $rc done } function create_snapshot(){ local snap="$1" log info "VM $vm_id - Creating snapshot $snap" - if ! do_run "rbd snap create $snap"; then - return 1; - fi + do_run "rbd snap create $snap" + rc=$? + log debug "create_snapshot() return $rc" + return $rc } function vm_freeze() {