Add TPM Disk to migration

README.md

@@ -2,7 +2,7 @@
 
 [![License](https://img.shields.io/github/license/EnterpriseVE/eve4pve-barc.svg)](https://www.gnu.org/licenses/gpl-3.0.en.html)
 
-Cross-Pool (live) Replication and near-live migration for Proxmox VE
+Cross-Pool asynchronous online-replication and near-live migration for Proxmox VE
 
 
 ```text
@@ -11,7 +11,7 @@ ______
 |   --|  _| . |_ -|_ -| . | | | -_|  _|
 |_____|_| |___|___|___|___|\_/|___|_|  
 
-Cross Pool (live) replication and near-live migration for Proxmox VE  
+Cross Pool asynchronous online-replication and near-live migration for Proxmox VE  
 
 Usage:
     crossover <COMMAND> [ARGS] [OPTIONS]
@@ -25,6 +25,7 @@ Commands:
     mirror               Replicate a stopped VM to another Cluster (full clone)
 
 Options:
+    --sshcipher          SSH Cipher to use for transfer (default: aes128-gcm@openssh.com,aes128-cbc)
     --vmid               The source+target ID of the VM, comma separated (eg. --vmid=100:100,101:101)
                          (The possibility to specify a different Target VMID is to not interfere with VMIDs on the
                          target cluster, or mark mirrored VMs on the destination)
@@ -90,7 +91,8 @@ It'll work according this scheme:
 
 ## Installation of prerequisites
 
-```apt install git pv gawk jq
+```
+apt install git pv gawk jq curl
 
 ## Install the Script somewhere, eg to /opt
 
@@ -222,13 +224,13 @@ Differential Bytes .......: 4.37 MiB
 
 ## Things to check
 
-From Proxmox VE Hosts you want to backup you need to be able to ssh passwordless to all other Cluster hosts, that may hold VM's or Containers. This goes for the source and for the destination Cluster.
+From Proxmox VE Hosts you want to backup you need to be able to ssh passwordless to all other Cluster hosts, that may hold VM's or Containers. This goes for the source and for the destination Cluster. Doublecheck this.
 
 This is required for using the free/unfreeze and the lock/unlock function, which has to be called locally from that Host the guest is currently running on. Usually this works out of the box for the source cluster, but you may want to make sure that you can "ssh root@pvehost1...n" from every host to every other host in the cluster.
 
-For the Destination Cluster you need to copy your ssh-key to the first host in the cluster, and login once to every node
+For the Destination Cluster you need to copy your ssh-key to the first host in the cluster, and login once to every node in your cluster.
-in your cluster.
 
+Currently preflight checks don't include the check for enough resources in the destination cluster. Check beforehand that you don't exceed the maximum safe size of ceph in the destination cluster.
 
 ## Some words about Snapshot consistency and what qemu-guest-agent can do for you
 
@@ -370,4 +372,5 @@ Ceph Documentation:
 [rdb – manage rados block device (rbd) images](http://docs.ceph.com/docs/master/man/8/rbd/)
 
 Proxmox Wiki:
-https://pve.proxmox.com/wiki/
+https://pve.proxmox.com/wiki/
+

crossover

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Cross Pool Migration and incremental replication Tool for Proxmox VMs using Ceph.
+# Author: Bastian Mäuser <bma@netz.org>        
+
 LC_ALL="en_US.UTF-8"
 
 source rainbow.sh
@@ -13,13 +16,11 @@ declare opt_influx_jobname=''
 declare opt_influx_job_metrics='crossover_xmit'
 declare opt_influx_summary_metrics='crossover_jobs'
 
-# Cross Pool Migration and incremental replication Tool for Proxmox VMs using Ceph.
+name=$(basename "$0")
-# Author: Bastian Mäuser <bma@netz.org>        
+# readonly variables
-
+declare -r NAME=$name
-declare -r VERSION=0.7
+declare -r VERSION=0.9
-declare -r NAME=$(basename "$0")
 declare -r PROGNAME=${NAME%.*}
-
 declare -r PVE_DIR="/etc/pve"
 declare -r PVE_NODES="$PVE_DIR/nodes"
 declare -r QEMU='qemu-server'
@@ -27,14 +28,21 @@ declare -r QEMU_CONF_CLUSTER="$PVE_NODES/*/$QEMU"
 declare -r EXT_CONF='.conf'
 declare -r PVFORMAT_FULL='e:%t r:%e c:%r a:%a %b %p'
 declare -r PVFORMAT_SNAP='e:%t c:%r a:%a %b'
+logfile=$(mktemp)
+declare -r LOG_FILE=$logfile
 
-declare -r LOG_FILE=$(mktemp)
+# associative global arrays 
-
 declare -A -g pvnode
 declare -A -g dstpvnode
 declare -A -g svmids
 declare -A -g dvmids
 
+declare -g scluster
+declare -g dcluster
+declare -g scephversion
+declare -g dcephversion
+
+# global integers
 declare -g -i perf_freeze_ok=0
 declare -g -i perf_freeze_failed=0
 declare -g -i perf_ss_ok=0
@@ -51,11 +59,17 @@ declare -g -i perf_bytes_total=0
 declare -g -i perf_vm_running=0
 declare -g -i perf_vm_stopped=0
 declare -g -i perf_snaps_removed=0
+declare -g -i perf_vm_total=0
+declare -g -i perf_vm_ok=0
 
+# commandline parameters
 declare opt_destination
 declare opt_vm_ids=''
 declare opt_snapshot_prefix='mirror-'
 declare opt_rewrite=''
+declare opt_pool='rbd'
+declare opt_sshcipher='aes128-gcm@openssh.com,aes128-cbc'
+declare opt_tag=''
 declare -i opt_prefix_id
 declare opt_exclude_vmids=''
 declare -i opt_debug=0
@@ -67,6 +81,7 @@ declare -i opt_keepdlock=0
 declare -i opt_overwrite=0
 declare -i opt_online=0
 declare -i opt_migrate=0
+declare -i opt_noconfirm=0
 declare opt_keep_local='0s'
 declare opt_keep_remote='0s'
 
@@ -75,6 +90,7 @@ declare -r redstconf='^\/etc\/pve\/nodes\/(.*)\/qemu-server\/([0-9]+).conf$'
 declare -r recephimg='([a-zA-Z0-9]+)\:(.*)'
 declare -r restripsnapshots='/^$/,$d'
 declare -r redateex='^([0-9]{4})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})$'
+declare -r restripansicolor='s/\x1b\[[0-9;]*m//g'
 
 function usage(){
    shift
@@ -104,11 +120,13 @@ Commands:
     mirror               Replicate a stopped VM to another Cluster (full clone)
 
 Options:
+    --sshcipher          SSH Cipher to use for transfer (default: aes128-gcm@openssh.com,aes128-cbc)
+    --tag                Include all VMs with a specific tag set in the Proxmox UI (if set, implies vmid=all)
     --vmid               The source+target ID of the VM/CT, comma separated (eg. --vmid=100:100,101:101), or all for all
     --prefixid           Prefix for VMID's on target System [optional]
     --excludevmids       Exclusde VM IDs when using --vmid==all
     --destination        Target PVE Host in target pool. e.g. --destination=pve04
-    --pool               Ceph pool name in target pool. e.g. --pool=data
+    --pool               Ceph pool name in target pool. e.g. --pool=data [default=rbd]
     --keeplocal          How many additional Snapshots to keep locally. e.g. --keeplocal=2d
     --keepremote         How many additional Snapshots to keep remote. e.g. --keepremote=7d
     --rewrite            PCRE Regex to rewrite the Config Files (eg. --rewrite='s/(net0:)(.*)tag=([0-9]+)/\1\2tag=1/g' would
@@ -117,19 +135,20 @@ Options:
     --influxtoken        Influx API token with write permission
     --influxbucket       Influx Bucket to write to (e.g. --influxbucket=telegraf/autogen)
     --jobname            Descriptive name for the job, used in Statistics
+    --mail               Mail address to send report to, comma-seperated (e.g. --mail=admin@test.com,admin2@test.com)
 Switches:
     --online             Allow online Copy
-    --migrate            Stop VM on Source Cluster before final Transfer and restart on destination Cluster
+    --migrate            Stop VM on Source Cluster before final Transfer and start on destination Cluster
     --nolock             Don't lock source VM on Transfer (mainly for test purposes)
     --keep-slock         Keep source VM locked on Transfer
     --keep-dlock         Keep VM locked after transfer on Destination
     --overwrite          Overwrite Destination
+    --noconfirm          Don't ask for confirmation before starting --migrate mode (use with care!)
     --debug              Show Debug Output
 
 Report bugs to <mephisto@mephis.to>
 
 EOF
-    exit 1
 }
 
 function parse_opts(){
@@ -139,7 +158,7 @@ function parse_opts(){
     local args
     args=$(getopt \
            --options '' \
-           --longoptions=vmid:,prefixid:,excludevmids:,destination:,pool:,keeplocal:,keepremote:,rewrite:,influxurl:,influxorg:,influxtoken:,influxbucket:,jobname:,online,migrate,nolock,keep-slock,keep-dlock,overwrite,dry-run,debug \
+           --longoptions=sshcipher:,tag:,vmid:,prefixid:,excludevmids:,destination:,pool:,keeplocal:,keepremote:,rewrite:,influxurl:,influxorg:,influxtoken:,influxbucket:,jobname:,mail:,online,migrate,nolock,keep-slock,keep-dlock,overwrite,dry-run,noconfirm,debug,syslog \
            --name "$PROGNAME" \
            -- "$@") \
            || end_process 128
@@ -148,6 +167,8 @@ function parse_opts(){
 
     while true; do    
       case "$1" in
+        --sshcipher) opt_sshcipher=$2; shift 2;;
+        --tag) opt_tag=$2; shift 2;;
         --vmid) opt_vm_ids=$2; shift 2;;
         --prefixid) opt_prefix_id=$2; shift 2;;
         --excludevmids) opt_exclude_vmids=$2; shift 2;;
@@ -161,14 +182,17 @@ function parse_opts(){
         --influxtoken) opt_influx_token=$2; shift 2;;
         --influxbucket) opt_influx_bucket=$2; shift 2;;
         --jobname) opt_influx_jobname=$2; shift 2;;
+        --mail) opt_addr_mail="$2"; shift 2;;
         --online) opt_online=1; shift ;;
         --migrate) opt_migrate=1; shift ;;
         --dry-run) opt_dry_run=1; shift;;
+        --noconfirm) opt_noconfirm=1; shift;;
         --debug) opt_debug=1; shift;;
         --nolock) opt_lock=0; shift;;
         --keep-slock) opt_keepslock=1; shift;;
         --keep-dlock) opt_keepdlock=1; shift;;
         --overwrite) opt_overwrite=1; shift;;
+        --syslog) opt_syslog=1; shift;;
         --) shift; break;;
         *) break;;
       esac
@@ -181,12 +205,11 @@ function parse_opts(){
         log info "============================================"
         log info "Proxmox VE Version:"
 
-        echowhite $(pveversion)
+        echowhite "$(pveversion)"
 
         log info "============================================"
     fi
 
-    [ -z "$opt_vm_ids" ] && { log info "VM id is not set."; end_process 1; }
 
     [ -z "$opt_influx_jobname" ] && { log info "Jobname is not set."; end_process 1; }
 
@@ -209,6 +232,15 @@ function parse_opts(){
         end_process 255
     fi
 
+    if [ -n "$opt_tag" ] && [ -n "$opt_vm_ids" ] && [ "$opt_vm_ids" != "all" ]; then
+        log error "You can't use --tag and --vmid at the same time"
+        end_process 255
+    fi
+
+    [ -n "$opt_tag" ] && [ -z $opt_vm_ids ] && opt_vm_ids="all"
+
+    [ -z "$opt_vm_ids" ] && { log info "VM id is not set."; end_process 1; }
+
     if [ "$opt_vm_ids" = "all" ]; then
         local all=''
         local data=''
@@ -216,6 +248,7 @@ function parse_opts(){
         local ids=''
 
         all=$(get_vm_ids "$QEMU_CONF_CLUSTER/*$EXT_CONF" "$LXC_CONF_CLUSTER/*$EXT_CONF")
+        log debug "all: $all"
         all=$(echo "$all" | tr ',' "\n")
         opt_exclude_vmids=$(echo "$opt_exclude_vmids" | tr ',' "\n")
         for id in $all; do
@@ -236,7 +269,7 @@ function parse_opts(){
             vm_ids=$(echo "$opt_vm_ids" | tr ',' "\n")
         fi
     fi
-    log debug "vm_ids: $vm_ids"
+    
 }
 
 human_readable() {
@@ -280,7 +313,14 @@ function exist_file(){
 function lookupcephpool() {
     pvehost=$1
     pvepoolname=$2
-    res=$(ssh $pvehost cat /etc/pve/storage.cfg | sed -n "/rbd: $pvepoolname/,/^$/p" | grep pool | cut -d " " -f 2)
+    res=$(ssh $pvehost cat /etc/pve/storage.cfg | sed -n "/rbd: $pvepoolname/,/^$/p" | grep -E "\s+pool\s" | cut -d " " -f 2)
+    echo $res
+}
+
+function lookupdatapool() {
+    pvehost=$1
+    pvepoolname=$2
+    res=$(ssh $pvehost cat /etc/pve/storage.cfg | sed -n "/rbd: $pvepoolname/,/^$/p" | grep -E "\s+data-pool\s" | cut -d " " -f 2)
     echo $res
 }
 
@@ -291,7 +331,9 @@ function get_vm_ids(){
     while [ $# -gt 0 ]; do
         for conf in $1; do
             [ ! -e "$conf" ] && break
-
+            if [ -n "$opt_tag" ]  && ! grep -qE "^tags:\s.*$opt_tag(;|$)" $conf; then
+                continue
+            fi
             conf=$(basename "$conf")
             [ "$data" != '' ] && data="$data,"
             data="$data${conf%.*}"
@@ -302,20 +344,6 @@ function get_vm_ids(){
     echo "$data"
 }
 
-function get_config_file(){
-    local file_config=''
-
-    if exist_file "$QEMU_CONF_CLUSTER/$vm_id$EXT_CONF"; then
-        file_config=$(ls $QEMU_CONF_CLUSTER/$vm_id$EXT_CONF)
-
-    else
-        log error "VM $vm_id - Unknown technology or VMID not found: $QEMU_CONF_CLUSTER/$vm_id$EXT_CONF"
-        end_process 128
-    fi 
-
-    echo "$file_config"
-}
-
 function get_disks_from_config(){
     local disks;
     local file_config=$1
@@ -327,7 +355,7 @@ function get_disks_from_config(){
                 [[ "$line" == "" ]] && break
                 echo "$line" 
             done < "$file_config" | \
-            grep -P '^(?:((?:virtio|ide|scsi|sata|mp)\d+)|rootfs): ' | \
+            grep -P '^(?:((?:efidisk|virtio|ide|scsi|sata|mp|tpmstate)\d+)|rootfs): ' | \
             grep -v -P 'cdrom|none' | \
             grep -v -P 'backup=0' | \
             awk '{ split($0,a,","); split(a[1],b," "); print b[2]}')
@@ -339,6 +367,7 @@ function log(){
     local level=$1
     shift 1
     local message=$*
+    local syslog_msg=''
 
     case $level in
         debug) 
@@ -350,28 +379,32 @@ function log(){
 
         info) 
             echo -e "$message"; 
-            echo -e "$message" >> "$LOG_FILE";
+            echo -e "$message" | sed -e 's/\x1b\[[0-9;]*m//g' >> "$LOG_FILE";
-            [ $opt_syslog -eq 1 ] && logger -t "$PROGNAME" "$message"
+            syslog_msg=$(echo -e "$message" | sed -e ${restripansicolor})
+            [ $opt_syslog -eq 1 ] && logger -t "$PROGNAME" "$syslog_msg"
             ;;
 
         warn)
-            echo -n $(echoyellow "WARNING: ")
+            echo -n "$(echoyellow 'WARNING: ')"
-            echo $(echowhite "$message") 1>&2
+            echowhite "$message" 1>&2
-            echo -e "$message" >> "$LOG_FILE";            
+            echo -e "$message" | sed -e ${restripansicolor} >> "$LOG_FILE";            
-            [ $opt_syslog -eq 1 ] && logger -t "$PROGNAME" -p daemon.warn "$message"
+            syslog_msg=$(echo -e "$message" | sed -e ${restripansicolor})
+            [ $opt_syslog -eq 1 ] && logger -t "$PROGNAME" -p daemon.warn "$syslog_msg"
             ;;
         
         error)
-            echo -n $(echored "ERROR: ")
+            echo -n "$(echored 'ERROR: ')"
-            echo $(echowhite "$message") 1>&2
+            echowhite "$message" 1>&2
-            echo -e "$message" >> "$LOG_FILE";            
+            echo -e "$message" | sed -e ${restripansicolor} >> "$LOG_FILE";            
-            [ $opt_syslog -eq 1 ] && logger -t "$PROGNAME" -p daemon.err "$message"
+            syslog_msg=$(echo -e "$message" | sed -e ${restripansicolor})
+            [ $opt_syslog -eq 1 ] && logger -t "$PROGNAME" -p daemon.err "$syslog_msg"
             ;;
 
         *)  
             echo "$message" 1>&2
-            echo -e "$message" >> "$LOG_FILE";            
+            echo -e "$message" | sed -e ${restripansicolor} >> "$LOG_FILE";            
-            [ $opt_syslog -eq 1 ] && logger -t "$PROGNAME" "$message"
+            syslog_msg=$(echo -e "$message" | sed -e ${restripansicolor})
+            [ $opt_syslog -eq 1 ] && logger -t "$PROGNAME" "$syslog_msg"
             ;;
     esac
 }
@@ -403,6 +436,9 @@ function mirror() {
     local -i endjob
     local -i vmcount=0
     local -i diskcount=0
+    local -i startdowntime
+    local -i enddowntime
+    local -i ga_ping
 
     local disp_perf_freeze_failed
     local disp_perf_ss_failed
@@ -413,6 +449,9 @@ function mirror() {
     log info "Start mirror $(date "+%F %T")"
     startjob=$(date +%s)
 
+    get_ceph_version
+    log info "Local Ceph Version: $scephversion, Remote Ceph Version: $dcephversion"
+
     #create pid file
     local pid_file="/var/run/$PROGNAME.pid"
     if [[ -e "$pid_file" ]]; then
@@ -427,37 +466,61 @@ function mirror() {
         end_process 1
     fi
 
-    
+    scluster=$(grep cluster_name /etc/pve/corosync.conf | cut -d " " -f 4)
+    dcluster=$(ssh "$opt_destination" grep cluster_name /etc/pve/corosync.conf | cut -d " " -f 4)
+
+    if [ $opt_migrate -eq 1 ] && [ $opt_noconfirm -eq 0 ]; then
+        echo "VM(s) $opt_vm_ids will subsequently be shutdown on [$scluster] and started on [$dcluster]"
+        read -p "Do you want to proceed? (yes/no) " yn
+        case $yn in 
+            yes ) echo ok, we will proceed;;
+            no ) echo exiting...;
+                exit;;
+            * ) echo invalid response;
+                exit 1;;
+        esac
+    fi
+
+
 
     map_source_to_destination_vmid
     map_vmids_to_host
     map_vmids_to_dsthost "$opt_destination"
 
-    if [ $(check_pool_exist "$opt_pool") -eq 0 ]; then
+    if [ "$(check_pool_exist "$opt_pool")" -eq 0 ]; then
         log error "Preflight check: Destination RBD-Pool $opt_pool does not exist."
         end_process 255
     fi
 
     for vm_id in $svmids; do
+        file_config="$PVE_NODES/${pvnode[$vm_id]}/$QEMU/$vm_id.conf"
+        if ! exist_file "$file_config"; then
+            log error "VM $vm_id - Preflight check: VM $vm_id does not exist on source cluster [$scluster] - skipping to next VM."
+            continue
+        fi
+        ga_ping=$(gaping "$vm_id")
+        log debug "ga_ping: $ga_ping"
+        if [ "$ga_ping" -eq 255 ] ; then    #vm running but no qemu-guest-agent answering
+            log error "VM $vm_id - Preflight check: VM $vm_id on source cluster [$scluster] has no qemu-guest-agent running - skipping to next VM."
+            continue
+        fi
         (( vmcount++ ))
-        local file_config; file_config=$(get_config_file)
-        [ -z "$file_config" ] && continue
         local disk=''
         dvmid=${dvmids[$vm_id]}
         vmname=$(cat $PVE_NODES/"${pvnode[$vm_id]}"/$QEMU/"$vm_id".conf | sed -e ''$restripsnapshots'' | grep "name\:" | cut -d' ' -f 2)
         log info "VM $vm_id - Starting mirror for $(echowhite "$vmname")"
         srcvmgenid=$(cat $PVE_NODES/"${pvnode[$vm_id]}"/$QEMU/"$vm_id".conf | sed -e ''$restripsnapshots'' | grep vmgenid | sed -r -e 's/^vmgenid:\s(.*)/\1/')
-        dstvmgenid=$(ssh $opt_destination cat $PVE_NODES/"${dstpvnode[$dvmid]}"/$QEMU/"$dvmid".conf 2>/dev/null | grep vmgenid | sed -e ''$restripsnapshots'' | sed -r -e 's/^vmgenid:\s(.*)/\1/')
+        dstvmgenid=$(ssh "$opt_destination" cat $PVE_NODES/"${dstpvnode[$dvmid]}"/$QEMU/"$dvmid".conf 2>/dev/null | grep vmgenid | sed -e ''$restripsnapshots'' | sed -r -e 's/^vmgenid:\s(.*)/\1/')
         log info "VM $vm_id - Checking for VM $dvmid on destination cluster $opt_destination $QEMU_CONF_CLUSTER"
         log debug "DVMID:$dvmid srcvmgenid:$srcvmgenid dstvmgenid:$dstvmgenid"
-        conf_on_destination=$(ssh $opt_destination "ls -d $QEMU_CONF_CLUSTER/$dvmid$EXT_CONF 2>/dev/null")
+        conf_on_destination=$(ssh "$opt_destination" "ls -d $QEMU_CONF_CLUSTER/$dvmid$EXT_CONF 2>/dev/null")
         [[ "$conf_on_destination" =~ $redstconf ]]
         host_on_destination=${BASH_REMATCH[1]}
 
         if [ $host_on_destination ]; then
             dststatus=$(ssh root@${dstpvnode[$dvmid]} qm status $dvmid|cut -d' ' -f 2)
             if [ $dststatus == "running" ]; then
-                log error "VM is running on Destination. bailing out"
+                log error "VM is running on Destination Cluster [$dcluster]. bailing out"
                 end_process 255
             fi
         fi
@@ -473,7 +536,7 @@ function mirror() {
                 log error "Source VM genid ($srcvmgenid) doesn't match destination VM genid ($dstvmgenid). This should not happen. Bailing out.."
                 end_process 255
             fi
-            log info "VM $vm_id - Transmitting Config for to destination $opt_destination VMID $dvmid"
+            log info "VM $vm_id - Transmitting Config for VM $vm_id to destination $opt_destination VMID $dvmid"
             rewriteconfig $PVE_NODES/"${pvnode[$vm_id]}"/$QEMU/"$vm_id".conf $opt_destination "$opt_pool" $PVE_NODES/"$opt_destination"/$QEMU/"$dvmid".conf "$dvmid"
             map_vmids_to_dsthost "$opt_destination"
         fi
@@ -486,21 +549,22 @@ function mirror() {
                 do_run "ha-manager remove $vm_id"
             fi
             do_run "ssh root@${pvnode[$vm_id]} qm shutdown $vm_id >/dev/null"
+            startdowntime=$(date +%s)
         fi
 
         #Lock on source + destination
         if [ $opt_lock -eq 1 ]; then
-            do_run "ssh root@"${pvnode[$vm_id]}" qm set "$vm_id" --lock backup" >/dev/null
+            do_run "ssh root@""${pvnode[$vm_id]}"" qm set ""$vm_id"" --lock backup" >/dev/null
             log info "VM $vm_id - locked $vm_id [rc:$?] on source"
-            do_run "ssh root@"${dstpvnode[$dvmid]}" qm set "$dvmid" --lock backup" >/dev/null
+            do_run "ssh root@""${dstpvnode[$dvmid]}"" qm set ""$dvmid"" --lock backup" >/dev/null
             log info "VM $dvmid - locked $dvmid [rc:$?] on destination"
         fi
-        #Freeze fs only if no migration running
+        #Freeze fs only if no migration running and qemu-guest-agent okay.
-        if [ $opt_migrate -eq 0 ]; then
+        if [ $opt_migrate -eq 0 ] && [ $ga_ping -eq 0 ]; then
             vm_freeze "$vm_id" "${pvnode[$vm_id]}" >/dev/null
             freezerc=$?
             if [ $freezerc -gt 0 ]; then
-                log error "VM $vm_id - QEMU-Guest could not fsfreeze on guest."
+                log warn "VM $vm_id - QEMU-Guest could not fsfreeze on guest."
                 (( perf_freeze_failed++ ))
             else
                 (( perf_freeze_ok++ ))
@@ -511,7 +575,7 @@ function mirror() {
             create_snapshot "$src_image_spec@$opt_snapshot_prefix$timestamp" 2>/dev/null
             ssrc=$?
             if [ $ssrc -gt 0 ]; then
-                log error "VM $vm_id - rbd snap failed."
+                log warn "VM $vm_id - rbd snap failed."
                 (( perf_ss_failed++ ))
             else
                 (( perf_ss_ok++ ))
@@ -542,6 +606,10 @@ function mirror() {
             [[ $dst_image_spec =~ ^.*\/(.*)$ ]]
             dst_image_name=${BASH_REMATCH[1]}-$src_image_pool_pve
             dst_image_pool=$(lookupcephpool $opt_destination $opt_pool)
+            dst_data_pool=$(lookupdatapool $opt_destination $opt_pool)
+            if [ -n "$dst_data_pool" ]; then
+                dst_data_opt="--data-pool $dst_data_pool"
+            fi
             snapshot_name="@$opt_snapshot_prefix$timestamp"
             localsnapcount=$(rbd ls -l $src_image_pool | grep $src_image_name@$opt_snapshot_prefix | cut -d ' ' -f 1|wc -l)
             if [ $localsnapcount -ge 2 ]; then
@@ -565,7 +633,7 @@ function mirror() {
                 #snapts=$(echo $currentlocal | sed -r -e 's/.*@mirror-(.*)/\1/')
                 snapshotsize=$(rbd du --pretty-format --format json $src_image_pool/$src_image_name|jq '.images[] | select (.snapshot_id == null) | {provisioned_size}.provisioned_size'|tail -1)
                 log debug "snapsize: $snapshotsize "
-                xmitjob="rbd export --rbd-concurrent-management-ops 8 $src_image_pool/$src_image_name$snapshot_name --no-progress - | tee >({ wc -c; } >/tmp/$PROGNAME.$pid.$dst_image_pool-$dst_image_name.size) | pv -s $snapshotsize -F \"VM $vm_id - F $src_image_pool/$src_image_name$snapshot_name: $PVFORMAT_FULL\" | ssh $opt_destination rbd import --image-format 2 - $dst_image_pool/$dst_image_name  2>/dev/null"
+                xmitjob="rbd export --rbd-concurrent-management-ops 8 $src_image_pool/$src_image_name$snapshot_name --no-progress - | tee >({ wc -c; } >/tmp/$PROGNAME.$pid.$dst_image_pool-$dst_image_name.size) | pv -s $snapshotsize -F \"VM $vm_id - F $src_image_pool/$src_image_name$snapshot_name: $PVFORMAT_FULL\" | ssh -c $opt_sshcipher $opt_destination rbd import --image-format 2 - $dst_image_pool/$dst_image_name $dst_data_opt 2>/dev/null"
                 # create initial snapshot on destination
                 log debug "xmitjob: $xmitjob"
                 startdisk=$(date +%s)
@@ -593,7 +661,7 @@ function mirror() {
                     #disk was not attached, or really nothing has changed..
                     snapshotsize=0
                 fi
-                xmitjob="rbd export-diff --no-progress --from-snap $opt_snapshot_prefix$basets $src_image_pool/$currentlocal - | tee >({ wc -c; } >/tmp/$PROGNAME.$pid.$dst_image_pool-$dst_image_name.size) | pv -F \"VM $vm_id - I $src_image_pool/$src_image_name$snapshot_name: $PVFORMAT_SNAP\" | ssh $opt_destination rbd import-diff --no-progress - $dst_image_pool/$dst_image_name"
+                xmitjob="rbd export-diff --no-progress --from-snap $opt_snapshot_prefix$basets $src_image_pool/$currentlocal - | tee >({ wc -c; } >/tmp/$PROGNAME.$pid.$dst_image_pool-$dst_image_name.size) | pv -F \"VM $vm_id - I $src_image_pool/$src_image_name$snapshot_name: $PVFORMAT_SNAP\" | ssh -c $opt_sshcipher $opt_destination rbd import-diff --no-progress - $dst_image_pool/$dst_image_name"
                 log debug "xmitjob: $xmitjob"
                 startdisk=$(date +%s)
                 do_run "$xmitjob"
@@ -628,8 +696,10 @@ function mirror() {
         fi
         #--migrate so start on destination?
         if [ $opt_migrate -eq 1 ]; then
-            log info "VM $dvmid - Starting VM on ${pvnode[$vm_id]}"
+            log info "VM $dvmid - Starting VM on node ${dstpvnode[$dvmid]} in cluster [$dcluster]"
-            do_run "ssh root@"${dstpvnode[$dvmid]}" qm start "$dvmid >/dev/null
+            do_run "ssh root@""${dstpvnode[$dvmid]}"" qm start "$dvmid >/dev/null
+            enddowntime=$(date +%s)
+            log info "VM $dvmid - Downtime: $(( enddowntime - startdowntime )) Seconds"
         fi
 
     done
@@ -654,6 +724,8 @@ function mirror() {
         cmd="curl --request POST \"$opt_influx_api_url/v2/write?org=$opt_influx_api_org&bucket=$opt_influx_bucket&precision=ns\" --header \"Authorization: Token $opt_influx_token\" --header \"Content-Type: text/plain; charset=utf-8\" --header \"Accept: application/json\" --data-binary '$influxlp'"
         do_run "$cmd"
     fi
+    (( perf_vm_ok++ ))
+    end_process 0
 }
 
 function do_housekeeping(){
@@ -695,6 +767,15 @@ function do_housekeeping(){
     done
 }
 
+function gaping() {
+    local vmid=$1
+    local rc
+    cmd="ssh root@${pvnode[$vmid]} qm guest cmd $vmid ping >/dev/null 2>&1"
+    eval "$cmd"
+    rc=$?
+    echo $rc
+}
+
 function create_snapshot(){
     local snap="$1"
     log info "VM $vm_id - Creating snapshot $snap"
@@ -710,8 +791,8 @@ function vm_freeze() {
     status=$(ssh root@"$fhost" qm status "$fvm"|cut -d' ' -f 2)
     if ! [[ "$status" == "running" ]]; then
         log info "VM $fvm - Not running, skipping fsfreeze-freeze"
-        return
         (( perf_vm_stopped++ ))
+        return
     else 
         (( perf_vm_running++ ))
     fi
@@ -751,7 +832,7 @@ function rewriteconfig(){
     else
         sedcmd='sed -e /^$/,$d'
     fi
-    cat "$oldconfig" | sed -r -e "s/^(virtio|ide|scsi|sata|mp)([0-9]+):\s([a-zA-Z0-9]+):(.*)-([0-9]+)-disk-([0-9]+).*,(.*)$/\1\2: $newpool:\4-$newvmid-disk-\6-\3,\7/g" | $sedcmd | sed -e '/^$/,$d' | sed -e '/ide[0-9]:.*-cloudinit,media=cdrom.*/d' | grep -v "^parent:\s.*$" | ssh "$dst" "cat - >$newconfig"
+    cat "$oldconfig" | sed -r -e "s/^(efidisk|virtio|ide|scsi|sata|mp|tpmstate)([0-9]+):\s([a-zA-Z0-9]+):(.*)-([0-9]+)-disk-([0-9]+).*,(.*)$/\1\2: $newpool:\4-$newvmid-disk-\6-\3,\7/g" | $sedcmd | sed -e '/^$/,$d' | sed -e '/ide[0-9]:.*-cloudinit,media=cdrom.*/d' | grep -v "^parent:\s.*$" | ssh "$dst" "cat - >$newconfig"
 }
 
 function checkvmid(){
@@ -782,23 +863,25 @@ function do_run(){
 
 function end_process(){
     local -i rc=$1;
-#    if ! [[ -z "$startts" && -z "$endts" ]]; then
+    local -i runtime
-#        local -i runtime=$(expr $endts - $startts)
+    local -i bps
-#        local -i bps=$(expr $bytecount/$runtime)
+    local -i ss_total
-#    fi
+    local subject
-#    local subject="Ceph [VM:$vmok/$vmtotal SS:$snapshotok/$snapshottotal EX:$exportok/$exporttotal] [$(bytesToHuman "$bytecount")@$(bytesToHuman "$bps")/s]"
+    if ! [[ -z "$startjob" || -z "$endjob" ]]; then
-#    [ $rc != 0 ] && subject="$subject [ERROR]"
+        runtime=$(expr $endjob - $startjob)
+        bps=$(expr $perf_bytes_total/$runtime)
+    fi
+    ss_total=$(expr $perf_ss_ok + $perf_ss_failed)
+    subject="Crossover [VM:$perf_vm_ok/$vmcount SS:$perf_ss_ok/$ss_total]"
+    [ $rc != 0 ] && subject="[ERROR] $subject" || subject="[OK] $subject"
 
-    #send email
+    local mail;
-#    local mail;
+    for mail in $(echo "$opt_addr_mail" | tr "," "\n"); do
-#    local mailhead="Backup $imgcount Images in $vmcount VMs (Bytes: $bytecount)"
+        do_run "cat '$LOG_FILE' | mail -s '$subject' '$mail'"
-#    for mail in $(echo "$opt_addr_mail" | tr "," "\n"); do
+    done
-#        do_run "cat '$LOG_FILE' | mail -s '$subject' '$mail'"
-#    done
 
     #remove log
-#    rm "$LOG_FILE"
+    rm "$LOG_FILE"
-
     exit "$rc";
 }
 
@@ -833,6 +916,11 @@ function check_pool_exist() {
     echo $exists
 }
 
+function get_ceph_version() {
+    scephversion=$(ceph -v | cut -d " " -f 3)
+    dcephversion=$(ssh $opt_destination ceph -v | cut -d " " -f 3)
+}
+
 function main(){    
     [ $# = 0 ] && usage;