From 9b29489dc82914d66e5d8a73f8e77e3c650a5ce0 Mon Sep 17 00:00:00 2001 From: Bastian Date: Mon, 24 Oct 2022 13:37:37 +0200 Subject: [PATCH] Added estimation of Transfer times and amounts of data --- README.md | 31 ++++++++++++++++++------------- crossover | 24 ++++++++++++++++-------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index bb943f2..142efea 100644 --- a/README.md +++ b/README.md @@ -25,18 +25,20 @@ Commands: mirror Replicate a stopped VM to another Cluster (full clone) Options: - --vmid The source+target ID of the VM/CT, comma separated (eg. --vmid=100:100,101:101), - --destination 'Target PVE Host in target pool. e.g. --destination=pve04 - --pool 'Ceph pool name in target pool. e.g. --pool=data - --keeplocal 'How many additional Snapshots to keep locally. e.g. --keeplocal=2 - --keepremote 'How many additional Snapshots to keep remote. e.g. --keepremote=2 - --online 'Allow online Copy - --nolock 'Don't lock source VM on Transfer (mainly for test purposes) - --keep-slock 'Keep source VM locked on Transfer - --keep-dlock 'Keep VM locked after transfer on Destination - --overwrite 'Overwrite Destination - --protect 'Protect Ceph Snapshots - --debug 'Show Debug Output + --vmid The source+target ID of the VM, comma separated (eg. --vmid=100:100,101:101) + (The possibility to specify a different Target VMID is to not interfere with VMIDs on the + target cluster, or mark mirrored VMs on the destination) + --destination Target PVE Host in target pool. e.g. --destination=pve04 + --pool Ceph pool name in target pool. e.g. --pool=data + --keeplocal How many additional Snapshots to keep locally. e.g. --keeplocal=2 + --keepremote How many additional Snapshots to keep remote. e.g. --keepremote=2 + --online Allow online Copy + --nolock Don't lock source VM on Transfer (mainly for test purposes) + --keep-slock Keep source VM locked on Transfer + --keep-dlock Keep VM locked after transfer on Destination + --overwrite Overwrite Destination + --protect Protect Ceph Snapshots + --debug Show Debug Output Report bugs to the Github repo at https://github.com/lephisto/crossover/ ``` @@ -75,6 +77,7 @@ It'll work according this scheme: * Can keep multiple backup * Retention policy: (eg. keep x snapshots on the source and y snapshots in the destination cluster) * Rewrites VM configurations so they match the new VMID and/or poolname on the destination +* Secure an encrypted transfer (SSH), so it's safe to mirror between datacenter without an additional VPN ## Protected / unprotected snapshot @@ -85,7 +88,7 @@ it's not aware of that paramter. ## Installation of prerequisites -```apt install git +```apt install git pv ## Install the Script somewhere, eg to /opt @@ -93,6 +96,8 @@ git clone https://github.com/lephisto/crossover/ /opt ``` +Ensure that you can freely ssh from the Node you plan to mirror _from_ to _all_ nodes in the destination cluster, as well as localhost. + ## Usage Mirror VM to another Cluster: diff --git a/crossover b/crossover index 3db18e6..a928dcd 100755 --- a/crossover +++ b/crossover @@ -14,6 +14,7 @@ declare -r PVE_NODES="$PVE_DIR/nodes" declare -r QEMU='qemu-server' declare -r QEMU_CONF_CLUSTER="$PVE_NODES/*/$QEMU" declare -r EXT_CONF='.conf' +declare -r PVFORMAT='elapsed:%t remaining:%e current:%r average:%a %p' declare -r LOG_FILE=$(mktemp) @@ -330,7 +331,7 @@ function mirror() { log error "Source VM genid ($srcvmgenid) doesn't match destination VM genid ($dstvmgenid). This should not happen. Bailing out.." end_process 255 fi - log info "Transmitting Config for VM $vm_id to desination $dvmid" + log info "VM $vm_id - Transmitting Config for to destination $opt_destination VMID $dvmid" rewriteconfig $PVE_NODES/"${pvnode[$vm_id]}"/$QEMU/"$vm_id".conf $opt_destination "$opt_pool" $PVE_NODES/"$opt_destination"/$QEMU/"$dvmid".conf "$dvmid" map_vmids_to_dsthost "$opt_destination" fi @@ -359,14 +360,12 @@ function mirror() { dst_image_spec=$(echo $src_image_spec | sed -r -e "s/(.*\/[a-zA-Z0-9]+\-)([0-9]+)(\-[a-zA-Z0-9]+\-[0-9]+)/\1$dvmid\3/") [ -z "$dst_image_spec" ] && continue [[ $disk =~ $recephimg ]] - #src_image_pool=${BASH_REMATCH[1]} - src_image_pool_pve=${BASH_REMATCH[1]} + src_image_pool_pve=${BASH_REMATCH[1]} src_image_pool=$(lookupcephpool "localhost" ${BASH_REMATCH[1]}) src_image_name=${BASH_REMATCH[2]} [[ $dst_image_spec =~ ^.*\/(.*)$ ]] dst_image_name=${BASH_REMATCH[1]}-$src_image_pool_pve dst_image_pool=$(lookupcephpool $opt_destination $opt_pool) - echo "dst_image_pool: $dst_image_pool" snapshot_name="@$opt_snapshot_prefix$timestamp" localsnapcount=$(rbd ls -l $src_image_pool | grep $src_image_name@$opt_snapshot_prefix | cut -d ' ' -f 1|wc -l) if [ $localsnapcount -ge 2 ]; then @@ -386,7 +385,10 @@ function mirror() { fi if [ -z $basets ]; then log debug "No matching Snapshot found on destination - Full Copy $src_image_pool/$src_image_name$snapshot_name to $dst_image_pool/$dst_image_name" - xmitjob="rbd export --rbd-concurrent-management-ops 8 $src_image_pool/$src_image_name$snapshot_name --no-progress -|pv -b -r|ssh $opt_destination rbd import --image-format 2 - $dst_image_pool/$dst_image_name" + snapts=$(echo $currentlocal | sed -r -e 's/.*@mirror-(.*)/\1/') + snapshotsize=$(rbd du --pretty-format --format json $src_image_pool/$src_image_name|jq '.images[] | select (.snapshot_id == null) | {provisioned_size}.provisioned_size') + log debug "snapsize $snapname: $snapshotsize " + xmitjob="rbd export --rbd-concurrent-management-ops 8 $src_image_pool/$src_image_name$snapshot_name --no-progress - | pv -s $snapshotsize -F \"VM $vm_id - Full xmit: $PVFORMAT\" | ssh $opt_destination rbd import --image-format 2 - $dst_image_pool/$dst_image_name" # create initial snapshot on destination if ! do_run $xmitjob; then log error "Transmitting Image failed" @@ -396,7 +398,11 @@ function mirror() { do_run $cmd else log debug "Basecopy + snapshot on destination - let's just transfer the diff" - xmitjob="rbd export-diff --no-progress --from-snap $opt_snapshot_prefix$basets $src_image_pool/$currentlocal - | pv -b -r |ssh $opt_destination rbd import-diff --no-progress - $dst_image_pool/$dst_image_name" + snapts=$(echo $currentlocal | sed -r -e 's/.*@mirror-(.*)/\1/') + snapshotsize=$(rbd du --pretty-format --format json $src_image_pool/$src_image_name|jq '.images[] | select (.snapshot == '\"$opt_snapshot_prefix$snapts\"') | {used_size}.used_size') + log debug "snapsize $snapname: $snapshotsize " + xmitjob="rbd export-diff --no-progress --from-snap $opt_snapshot_prefix$basets $src_image_pool/$currentlocal - | pv -s $snapshotsize -F \"VM $vm_id - Snap xmit: $PVFORMAT\" | ssh $opt_destination rbd import-diff --no-progress - $dst_image_pool/$dst_image_name" + log debug "xmitjob: $xmitjob" if ! do_run $xmitjob; then log error "Transmitting Image failed" return 1 @@ -404,16 +410,18 @@ function mirror() { do_housekeeping "localhost" "$src_image_pool" "$src_image_name" $opt_keep_local do_housekeeping "$opt_destination" "$dst_image_pool" "$dst_image_name" $opt_keep_remote fi + unset basets done if [ ! $opt_keepslock -eq 1 ]; then ssh root@${pvnode[$vm_id]} qm unlock $vm_id - log info "Unlocking source VM $vm_id" + log info "VM $vm_id - Unlocking source VM $vm_id" fi if [ $opt_keepdlock -eq 0 ]; then ssh root@${dstpvnode[$dvmid]} qm unlock $dvmid - log info "Unlocking destination VM $dvmid" + log info "VM $dvmid - Unlocking destination VM $dvmid" fi done + log info "Start mirror $(date "+%F %T")" } function do_housekeeping(){