wip

README.md

@@ -2,7 +2,7 @@
 
 [![License](https://img.shields.io/github/license/EnterpriseVE/eve4pve-barc.svg)](https://www.gnu.org/licenses/gpl-3.0.en.html)
 
-Cross-Pool (live) Replication and near-live migration for Proxmox VE
+Cross-Pool asynchronous online-replication and near-live migration for Proxmox VE
 
 
 ```text
@@ -11,7 +11,7 @@ ______
 |   --|  _| . |_ -|_ -| . | | | -_|  _|
 |_____|_| |___|___|___|___|\_/|___|_|  
 
-Cross Pool (live) replication and near-live migration for Proxmox VE  
+Cross Pool asynchronous online-replication and near-live migration for Proxmox VE  
 
 Usage:
     crossover <COMMAND> [ARGS] [OPTIONS]
@@ -25,6 +25,7 @@ Commands:
     mirror               Replicate a stopped VM to another Cluster (full clone)
 
 Options:
+    --sshcipher          SSH Cipher to use for transfer (default: aes128-gcm@openssh.com,aes128-cbc)
     --vmid               The source+target ID of the VM, comma separated (eg. --vmid=100:100,101:101)
                          (The possibility to specify a different Target VMID is to not interfere with VMIDs on the
                          target cluster, or mark mirrored VMs on the destination)
@@ -90,7 +91,8 @@ It'll work according this scheme:
 
 ## Installation of prerequisites
 
-```apt install git pv gawk jq
+```
+apt install git pv gawk jq curl
 
 ## Install the Script somewhere, eg to /opt
 
@@ -222,13 +224,17 @@ Differential Bytes .......: 4.37 MiB
 
 ## Things to check
 
-From Proxmox VE Hosts you want to backup you need to be able to ssh passwordless to all other Cluster hosts, that may hold VM's or Containers. This goes for the source and for the destination Cluster.
+From Proxmox VE Hosts you want to backup you need to be able to ssh passwordless to all other Cluster hosts, that may hold VM's or Containers. This goes for the source and for the destination Cluster. Doublecheck this.
 
 This is required for using the free/unfreeze and the lock/unlock function, which has to be called locally from that Host the guest is currently running on. Usually this works out of the box for the source cluster, but you may want to make sure that you can "ssh root@pvehost1...n" from every host to every other host in the cluster.
 
-For the Destination Cluster you need to copy your ssh-key to the first host in the cluster, and login once to every node
+For the Destination Cluster you need to copy your ssh-key to the first host in the cluster, and login once to every node in your cluster.
-in your cluster.
 
+Currently preflight checks don't include the check for enough resources in the destination cluster. Check beforehand that you don't exceed the maximum safe size of ceph in the destination cluster.
+
+## Unique Disk names
+
+There are cases, when the Source VM has Disks on different ceph pools. Now, in theory you can have identical image names for different disks. Since all disk images are migrated to one destination pool, they need to be unique. This tool detects this in Preflight checks, and skips these VMs and issues a warning. To solve this, give them unique names, like vm-100-disk-0, vm,100-disk-1 and so on. `rbd mv` will help you.
 
 ## Some words about Snapshot consistency and what qemu-guest-agent can do for you
 
@@ -371,3 +377,4 @@ Ceph Documentation:
 
 Proxmox Wiki:
 https://pve.proxmox.com/wiki/
+

crossover

@@ -19,7 +19,7 @@ declare opt_influx_summary_metrics='crossover_jobs'
 name=$(basename "$0")
 # readonly variables
 declare -r NAME=$name
-declare -r VERSION=0.8
+declare -r VERSION=0.9
 declare -r PROGNAME=${NAME%.*}
 declare -r PVE_DIR="/etc/pve"
 declare -r PVE_NODES="$PVE_DIR/nodes"
@@ -68,6 +68,8 @@ declare opt_vm_ids=''
 declare opt_snapshot_prefix='mirror-'
 declare opt_rewrite=''
 declare opt_pool='rbd'
+declare opt_sshcipher='aes128-gcm@openssh.com,aes128-cbc'
+declare opt_tag=''
 declare -i opt_prefix_id
 declare opt_exclude_vmids=''
 declare -i opt_debug=0
@@ -118,6 +120,8 @@ Commands:
     mirror               Replicate a stopped VM to another Cluster (full clone)
 
 Options:
+    --sshcipher          SSH Cipher to use for transfer (default: aes128-gcm@openssh.com,aes128-cbc)
+    --tag                Include all VMs with a specific tag set in the Proxmox UI (if set, implies vmid=all)
     --vmid               The source+target ID of the VM/CT, comma separated (eg. --vmid=100:100,101:101), or all for all
     --prefixid           Prefix for VMID's on target System [optional]
     --excludevmids       Exclusde VM IDs when using --vmid==all
@@ -154,7 +158,7 @@ function parse_opts(){
     local args
     args=$(getopt \
            --options '' \
-           --longoptions=vmid:,prefixid:,excludevmids:,destination:,pool:,keeplocal:,keepremote:,rewrite:,influxurl:,influxorg:,influxtoken:,influxbucket:,jobname:,mail:,online,migrate,nolock,keep-slock,keep-dlock,overwrite,dry-run,noconfirm,debug,syslog \
+           --longoptions=sshcipher:,tag:,vmid:,prefixid:,excludevmids:,destination:,pool:,keeplocal:,keepremote:,rewrite:,influxurl:,influxorg:,influxtoken:,influxbucket:,jobname:,mail:,online,migrate,nolock,keep-slock,keep-dlock,overwrite,dry-run,noconfirm,debug,syslog \
            --name "$PROGNAME" \
            -- "$@") \
            || end_process 128
@@ -163,6 +167,8 @@ function parse_opts(){
 
     while true; do    
       case "$1" in
+        --sshcipher) opt_sshcipher=$2; shift 2;;
+        --tag) opt_tag=$2; shift 2;;
         --vmid) opt_vm_ids=$2; shift 2;;
         --prefixid) opt_prefix_id=$2; shift 2;;
         --excludevmids) opt_exclude_vmids=$2; shift 2;;
@@ -204,7 +210,6 @@ function parse_opts(){
         log info "============================================"
     fi
 
-    [ -z "$opt_vm_ids" ] && { log info "VM id is not set."; end_process 1; }
 
     [ -z "$opt_influx_jobname" ] && { log info "Jobname is not set."; end_process 1; }
 
@@ -227,6 +232,15 @@ function parse_opts(){
         end_process 255
     fi
 
+    if [ -n "$opt_tag" ] && [ -n "$opt_vm_ids" ] && [ "$opt_vm_ids" != "all" ]; then
+        log error "You can't use --tag and --vmid at the same time"
+        end_process 255
+    fi
+
+    [ -n "$opt_tag" ] && [ -z $opt_vm_ids ] && opt_vm_ids="all"
+
+    [ -z "$opt_vm_ids" ] && { log info "VM id is not set."; end_process 1; }
+
     if [ "$opt_vm_ids" = "all" ]; then
         local all=''
         local data=''
@@ -234,6 +248,7 @@ function parse_opts(){
         local ids=''
 
         all=$(get_vm_ids "$QEMU_CONF_CLUSTER/*$EXT_CONF" "$LXC_CONF_CLUSTER/*$EXT_CONF")
+        log debug "all: $all"
         all=$(echo "$all" | tr ',' "\n")
         opt_exclude_vmids=$(echo "$opt_exclude_vmids" | tr ',' "\n")
         for id in $all; do
@@ -254,7 +269,7 @@ function parse_opts(){
             vm_ids=$(echo "$opt_vm_ids" | tr ',' "\n")
         fi
     fi
-    log debug "vm_ids: $vm_ids"
+    
 }
 
 human_readable() {
@@ -298,7 +313,14 @@ function exist_file(){
 function lookupcephpool() {
     pvehost=$1
     pvepoolname=$2
-    res=$(ssh $pvehost cat /etc/pve/storage.cfg | sed -n "/rbd: $pvepoolname/,/^$/p" | grep pool | cut -d " " -f 2)
+    res=$(ssh $pvehost cat /etc/pve/storage.cfg | sed -n "/rbd: $pvepoolname/,/^$/p" | grep -E "\s+pool\s" | cut -d " " -f 2)
+    echo $res
+}
+
+function lookupdatapool() {
+    pvehost=$1
+    pvepoolname=$2
+    res=$(ssh $pvehost cat /etc/pve/storage.cfg | sed -n "/rbd: $pvepoolname/,/^$/p" | grep -E "\s+data-pool\s" | cut -d " " -f 2)
     echo $res
 }
 
@@ -309,7 +331,9 @@ function get_vm_ids(){
     while [ $# -gt 0 ]; do
         for conf in $1; do
             [ ! -e "$conf" ] && break
-
+            if [ -n "$opt_tag" ]  && ! grep -qE "^tags:\s.*$opt_tag(;|$)" $conf; then
+                continue
+            fi
             conf=$(basename "$conf")
             [ "$data" != '' ] && data="$data,"
             data="$data${conf%.*}"
@@ -331,7 +355,7 @@ function get_disks_from_config(){
                 [[ "$line" == "" ]] && break
                 echo "$line" 
             done < "$file_config" | \
-            grep -P '^(?:((?:efidisk|virtio|ide|scsi|sata|mp)\d+)|rootfs): ' | \
+            grep -P '^(?:((?:efidisk|virtio|ide|scsi|sata|mp|tpmstate)\d+)|rootfs): ' | \
             grep -v -P 'cdrom|none' | \
             grep -v -P 'backup=0' | \
             awk '{ split($0,a,","); split(a[1],b," "); print b[2]}')
@@ -339,6 +363,30 @@ function get_disks_from_config(){
     echo "$disks"
 }
 
+function check_unique_disk_config() {
+    local file_config=$1
+    disks=$(while read -r line; do
+                [[ "$line" == "" ]] && break
+                echo "$line" 
+            done < "$file_config" | \
+            grep -P '^(?:((?:efidisk|virtio|ide|scsi|sata|mp|tpmstate)\d+)|rootfs): ' | \
+            grep -v -P 'cdrom|none' | \
+            grep -v -P 'backup=0' | \
+            awk '{ split($0,a,","); split(a[1],b," "); print b[2]}'| wc -l)
+    echo disks            
+    uniquedisks=$(while read -r line; do
+                [[ "$line" == "" ]] && break
+                echo "$line" 
+            done < "$file_config" | \
+            grep -P '^(?:((?:efidisk|virtio|ide|scsi|sata|mp|tpmstate)\d+)|rootfs): ' | \
+            grep -v -P 'cdrom|none' | \
+            grep -v -P 'backup=0' | \
+            awk '{ split($0,a,","); split(a[1],b," "); print b[2]}'|cut -d ':' -f 2 | sort -nr | uniq | wc -l)
+    # TBD: ^(vm|ct)-([0-9]+)-([a-z]+)-[\d]+.*$
+    difference=$(expr $disks - $uniquedisks)
+    echo "$difference"
+}
+
 function log(){
     local level=$1
     shift 1
@@ -412,6 +460,8 @@ function mirror() {
     local -i endjob
     local -i vmcount=0
     local -i diskcount=0
+    local -i vmdiskcount=0
+    local -i skipped_vm_count=0
     local -i startdowntime
     local -i enddowntime
     local -i ga_ping
@@ -467,17 +517,25 @@ function mirror() {
         log error "Preflight check: Destination RBD-Pool $opt_pool does not exist."
         end_process 255
     fi
-
     for vm_id in $svmids; do
         file_config="$PVE_NODES/${pvnode[$vm_id]}/$QEMU/$vm_id.conf"
+        check_unique_disk_config "$file_config"
+        end_process 255
+        if [[ $(check_unique_disk_config "$file_config") -ge 1 ]]; then
+            log error "VM $vm_id - Preflight check: VM $vm_id has duplicate disk entries - skipping to next VM. Check Documentation to learn how to avoid this."
+            (( skipped_vm_count++ ))
+            continue
+        fi
         if ! exist_file "$file_config"; then
             log error "VM $vm_id - Preflight check: VM $vm_id does not exist on source cluster [$scluster] - skipping to next VM."
+            (( skipped_vm_count++ ))
             continue
         fi
         ga_ping=$(gaping "$vm_id")
         log debug "ga_ping: $ga_ping"
         if [ "$ga_ping" -eq 255 ] ; then    #vm running but no qemu-guest-agent answering
             log error "VM $vm_id - Preflight check: VM $vm_id on source cluster [$scluster] has no qemu-guest-agent running - skipping to next VM."
+            (( skipped_vm_count++ ))
             continue
         fi
         (( vmcount++ ))
@@ -570,18 +628,23 @@ function mirror() {
         fi
         for disk in $(get_disks_from_config "$file_config"); do
             (( diskcount++ ))
-            log debug "VMID: $vm_id Disk: $disk DESTVMID: $dvmid"
+            (( vmdiskcount++ ))
             src_image_spec=$(get_image_spec "$disk")
+            log debug "src_image_spec: $src_image_spec"
             [ -z "$src_image_spec" ] && continue
             dst_image_spec=$(echo $src_image_spec | sed -r -e "s/(.*\/[a-zA-Z0-9]+\-)([0-9]+)(\-[a-zA-Z0-9]+\-[0-9]+)/\1$dvmid\3/")
             [ -z "$dst_image_spec" ] && continue
             [[ $disk =~ $recephimg ]]
-	        src_image_pool_pve=${BASH_REMATCH[1]}
+#	        src_image_pool_pve=${BASH_REMATCH[1]}
             src_image_pool=$(lookupcephpool "localhost" ${BASH_REMATCH[1]})
             src_image_name=${BASH_REMATCH[2]}
             [[ $dst_image_spec =~ ^.*\/(.*)$ ]]
-            dst_image_name=${BASH_REMATCH[1]}-$src_image_pool_pve
+            dst_image_name=${BASH_REMATCH[1]} #-$src_image_pool_pve
             dst_image_pool=$(lookupcephpool $opt_destination $opt_pool)
+            dst_data_pool=$(lookupdatapool $opt_destination $opt_pool)
+            if [ -n "$dst_data_pool" ]; then
+                dst_data_opt="--data-pool $dst_data_pool"
+            fi
             snapshot_name="@$opt_snapshot_prefix$timestamp"
             localsnapcount=$(rbd ls -l $src_image_pool | grep $src_image_name@$opt_snapshot_prefix | cut -d ' ' -f 1|wc -l)
             if [ $localsnapcount -ge 2 ]; then
@@ -605,7 +668,7 @@ function mirror() {
                 #snapts=$(echo $currentlocal | sed -r -e 's/.*@mirror-(.*)/\1/')
                 snapshotsize=$(rbd du --pretty-format --format json $src_image_pool/$src_image_name|jq '.images[] | select (.snapshot_id == null) | {provisioned_size}.provisioned_size'|tail -1)
                 log debug "snapsize: $snapshotsize "
-                xmitjob="rbd export --rbd-concurrent-management-ops 8 $src_image_pool/$src_image_name$snapshot_name --no-progress - | tee >({ wc -c; } >/tmp/$PROGNAME.$pid.$dst_image_pool-$dst_image_name.size) | pv -s $snapshotsize -F \"VM $vm_id - F $src_image_pool/$src_image_name$snapshot_name: $PVFORMAT_FULL\" | ssh $opt_destination rbd import --image-format 2 - $dst_image_pool/$dst_image_name  2>/dev/null"
+                xmitjob="rbd export --rbd-concurrent-management-ops 8 $src_image_pool/$src_image_name$snapshot_name --no-progress - | tee >({ wc -c; } >/tmp/$PROGNAME.$pid.$dst_image_pool-$dst_image_name.size) | pv -s $snapshotsize -F \"VM $vm_id - F $src_image_pool/$src_image_name$snapshot_name: $PVFORMAT_FULL\" | ssh -c $opt_sshcipher $opt_destination rbd import --image-format 2 - $dst_image_pool/$dst_image_name $dst_data_opt 2>/dev/null"
                 # create initial snapshot on destination
                 log debug "xmitjob: $xmitjob"
                 startdisk=$(date +%s)
@@ -633,7 +696,7 @@ function mirror() {
                     #disk was not attached, or really nothing has changed..
                     snapshotsize=0
                 fi
-                xmitjob="rbd export-diff --no-progress --from-snap $opt_snapshot_prefix$basets $src_image_pool/$currentlocal - | tee >({ wc -c; } >/tmp/$PROGNAME.$pid.$dst_image_pool-$dst_image_name.size) | pv -F \"VM $vm_id - I $src_image_pool/$src_image_name$snapshot_name: $PVFORMAT_SNAP\" | ssh $opt_destination rbd import-diff --no-progress - $dst_image_pool/$dst_image_name"
+                xmitjob="rbd export-diff --no-progress --from-snap $opt_snapshot_prefix$basets $src_image_pool/$currentlocal - | tee >({ wc -c; } >/tmp/$PROGNAME.$pid.$dst_image_pool-$dst_image_name.size) | pv -F \"VM $vm_id - I $src_image_pool/$src_image_name$snapshot_name: $PVFORMAT_SNAP\" | ssh -c $opt_sshcipher $opt_destination rbd import-diff --no-progress - $dst_image_pool/$dst_image_name"
                 log debug "xmitjob: $xmitjob"
                 startdisk=$(date +%s)
                 do_run "$xmitjob"
@@ -661,6 +724,7 @@ function mirror() {
                 do_run "$cmd"
             fi
             unset basets
+            vmdiskcount=0
         done
         if [ $opt_keepdlock -eq 0 ]; then
             ssh root@${dstpvnode[$dvmid]} qm unlock $dvmid
@@ -683,12 +747,14 @@ function mirror() {
     if [ "$perf_ss_failed" -gt 0 ]; then disp_perf_ss_failed="$(echored $perf_ss_failed)"; else disp_perf_ss_failed="$(echogreen $perf_ss_failed)"; fi
     if [ "$perf_full_failed" -gt 0 ]; then disp_perf_full_failed="$(echored $perf_full_failed)"; else disp_perf_full_failed="$(echogreen $perf_full_failed)"; fi
     if [ "$perf_diff_failed" -gt 0 ]; then disp_perf_diff_failed="$(echored $perf_diff_failed)"; else disp_perf_diff_failed="$(echogreen $perf_diff_failed)"; fi
+    if [ "$skipped_vm_count" -gt 0 ]; then disp_skipped_vm_count="$(echored $skipped_vm_count)"; else disp_skipped_vm_count="$(echogreen $skipped_vm_count)"; fi
     log info "VM Freeze OK/failed.......: $perf_freeze_ok/$disp_perf_freeze_failed"    
     log info "RBD Snapshot OK/failed....: $perf_ss_ok/$disp_perf_ss_failed"
     log info "RBD export-full OK/failed.: $perf_full_ok/$disp_perf_full_failed"
     log info "RBD export-diff OK/failed.: $perf_diff_ok/$disp_perf_diff_failed"
     log info "Full xmitted..............: $(human_readable $perf_bytes_full)"
     log info "Differential Bytes .......: $(human_readable $perf_bytes_diff)"
+    log info "Skipped VMs ..............: $disp_skipped_vm_count"
     if [ -n "$opt_influx_api_url" ]; then
         log info "VM $vm_id - Logging Job summary to InfluxDB: $opt_influx_api_url"
         influxlp="$opt_influx_summary_metrics,jobname=$opt_influx_jobname perf_bytes_diff=$perf_bytes_diff""i,perf_bytes_full=$perf_bytes_full""i,perf_bytes_total=$perf_bytes_total""i,perf_diff_failed=$perf_diff_failed""i,perf_diff_ok=$perf_diff_ok""i,perf_freeze_failed=$perf_freeze_failed""i,perf_freeze_ok=$perf_freeze_ok""i,perf_full_failed=$perf_full_failed""i,perf_full_ok=$perf_full_ok""i,perf_ss_failed=$perf_ss_failed""i,perf_ss_ok=$perf_ss_ok""i,perf_vm_running=$perf_vm_running""i,perf_vm_stopped=$perf_vm_stopped""i"
@@ -804,7 +870,7 @@ function rewriteconfig(){
     else
         sedcmd='sed -e /^$/,$d'
     fi
-    cat "$oldconfig" | sed -r -e "s/^(efidisk|virtio|ide|scsi|sata|mp)([0-9]+):\s([a-zA-Z0-9]+):(.*)-([0-9]+)-disk-([0-9]+).*,(.*)$/\1\2: $newpool:\4-$newvmid-disk-\6-\3,\7/g" | $sedcmd | sed -e '/^$/,$d' | sed -e '/ide[0-9]:.*-cloudinit,media=cdrom.*/d' | grep -v "^parent:\s.*$" | ssh "$dst" "cat - >$newconfig"
+    cat "$oldconfig" | sed -r -e "s/^(efidisk|virtio|ide|scsi|sata|mp)([0-9]+):\s([a-zA-Z0-9]+):(.*)-([0-9]+)-disk-([0-9]+).*,(.*)$/\1\2: $newpool:\4-$newvmid-disk-\6,\7/g" | $sedcmd | sed -e '/^$/,$d' | sed -e '/ide[0-9]:.*-cloudinit,media=cdrom.*/d' | grep -v "^parent:\s.*$" | ssh "$dst" "cat - >$newconfig"
 }
 
 function checkvmid(){