Update rbd to work with snapshots and virsh domblklist. Refactor images backup to newer code style.

kvm-backup-images.sh

@@ -12,7 +12,7 @@
 PIDFILE="/tmp/backup-image.pid"
 
 # If the pid file exists and process is running, exit.
-if [ -f "$PIDFILE" ]; then
+if [[ -f "$PIDFILE" ]]; then
     PID=$(cat "$PIDFILE")
     if ps -p "$PID" >/dev/null; then
         echo "Backup process already running, exiting."
@@ -36,13 +36,18 @@ export BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO
 # Set to empty string to disable pruning.
 PRUNE_OPTIONS="--keep-daily 7 --keep-weekly 4 --keep-monthly 6"
 
+# Remove PID file on exit.
+cleanup() {
+    rm "$PIDFILE"
+}
+trap cleanup EXIT
+
 # Allows providing an argument of a domain to specifically backup.
 BACKUP_DOMAIN="$1"
 
 # Failures should remove pid file and exit with status code 1.
 fail() {
     echo "$1"
-    rm "$PIDFILE"
     exit 1
 }
 
@@ -53,15 +58,14 @@ blockCommit() {
     DEV="$3"
     if [[ "$DOMSTATUS" == "running" ]]; then
         echo "Commit changes for $DOMAIN ($DEV)"
-        virsh blockcommit \
+
+        if ! virsh blockcommit \
                 "$DOMAIN" \
                 "$DEV" \
                 --active \
                 --verbose \
                 --pivot \
-            --delete
+                --delete; then
-
-        if [ $? -ne 0 ]; then
             fail "Could not commit changes $DOMAIN ($DEV). This may be a major issue and VM may be broken now."
         fi
     fi
@@ -69,18 +73,14 @@ blockCommit() {
 
 # I save the status in a temporary file so I can error out and exit if a failure occurs.
 DOMLIST_STATUS_TMP="/tmp/backup-image-domlist-tmp"
-while read -r line; do
+while read -r _ DOMAIN DOMSTATUS; do
-    # Extract the domain name and status from the line.
-    DOMAIN=$(echo $line | awk '{print $2}')
-    DOMSTATUS=$(echo $line | awk '{for (i=3; i<NF; i++) printf $i " "; if (NF>=3) print $NF}')
-
     # If the domain is empty, its not needed.
-    if [ -z "$DOMAIN" ]; then
+    if [[ -z "$DOMAIN" ]]; then
         continue
     fi
 
     # If a backup domain was provided, we're only going to backup that domain.
-    if [ -n "$BACKUP_DOMAIN" ] && [[ "$BACKUP_DOMAIN" != "$DOMAIN" ]]; then
+    if [[ -n "$BACKUP_DOMAIN" ]] && [[ "$BACKUP_DOMAIN" != "$DOMAIN" ]]; then
         continue
     fi
 
@@ -88,13 +88,9 @@ while read -r line; do
     DEVS=()
     IMAGES=()
     BLKLIST_STATUS_TMP="/tmp/backup-image-blklist-tmp"
-    while read -r line; do
+    while read -r DEV IMAGE; do
-        # Extract the device and image from the line.
-        DEV=$(echo $line | awk '{print $1}')
-        IMAGE=$(echo $line | awk '{for (i=2; i<NF; i++) printf $i " "; if (NF>=2) print $NF}')
-
         # Ignore empty line or no image.
-        if [ -z "$IMAGE" ] || [[ "$IMAGE" == "-" ]]; then
+        if [[ -z "$IMAGE" ]] || [[ "$IMAGE" == "-" ]]; then
             continue
         fi
 
@@ -103,23 +99,28 @@ while read -r line; do
             continue
         fi
 
+        # Ignore non-image files.
+        if ! [[ "$IMAGE" =~ ^\/ ]]; then
+            continue
+        fi
+
         # This image needs backing up.
         DEVS+=("$DEV")
         IMAGES+=("$IMAGE")
     done < <(
-        virsh domblklist $DOMAIN | tail -n +3
+        virsh domblklist "$DOMAIN" | tail -n +3
-        echo ${PIPESTATUS[0]} >$BLKLIST_STATUS_TMP
+        echo "${PIPESTATUS[0]}" >"$BLKLIST_STATUS_TMP"
     )
 
     # Get status from the block listing.
     status=1
-    if [ -f $BLKLIST_STATUS_TMP ]; then
+    if [[ -f $BLKLIST_STATUS_TMP ]]; then
-        status=$(cat $BLKLIST_STATUS_TMP)
+        status=$(cat "$BLKLIST_STATUS_TMP")
-        rm $BLKLIST_STATUS_TMP
+        rm "$BLKLIST_STATUS_TMP"
     fi
 
     # If status has an error, exit.
-    if [ $status -ne 0 ]; then
+    if ((status!=0)); then
         fail "Domain block listing failed"
     fi
 
@@ -135,7 +136,7 @@ while read -r line; do
         if [[ "$DOMSTATUS" == "running" ]]; then
             # If the snapshot file exists, we should commit changes before performing another snapshot.
             # We are assuming that we created the snapshot here, and that concurrent runs are not possible.
-            if [ -e "$IMAGESNAPSHOT" ]; then
+            if [[ -e "$IMAGESNAPSHOT" ]]; then
                 # Commit any blocks.
                 blockCommit "$DOMSTATUS" "$DOMAIN" "$DEV"
             fi
@@ -150,42 +151,36 @@ while read -r line; do
             fi
 
             echo "Creating snapshot for $DOMAIN ($DEV)"
-            virsh snapshot-create-as --domain "$DOMAIN" \
+            if ! virsh snapshot-create-as --domain "$DOMAIN" \
                     --name backup \
                     --no-metadata \
                     --atomic \
                     --disk-only \
-                --diskspec $DEV,snapshot=external
+                    --diskspec "$DEV,snapshot=external"; then
-
-            if [ $? -ne 0 ]; then
                 fail "Failed to create snapshot for $DOMAIN ($DEV)"
             fi
         fi
 
         # Backup the image.
         echo "Creating backup for $DOMAIN ($DEV [$IMAGE])"
-        pv "$IMAGE" | borg create \
+        if ! pv "$IMAGE" | borg create \
                 --verbose \
                 --stats \
                 --show-rc \
                 --stdin-name "$IMAGENAME" \
-            "::$DOMAIN-$DEV-{now}" -
+                "::$DOMAIN-$DEV-{now}" -; then
-
-        if [ $? -ne 0 ]; then
             # Commit any blocks.
             blockCommit "$DOMSTATUS" "$DOMAIN" "$DEV"
             fail "Failed to backup $DOMAIN ($DEV)"
         fi
 
         # Prune if options are configured.
-        if [ -n "$PRUNE_OPTIONS" ]; then
+        if [[ -n "$PRUNE_OPTIONS" ]]; then
             echo "Pruning backups for $DOMAIN ($DEV)"
-            borg prune --list \
+            if ! eval borg prune --list \
                     --show-rc \
-                --glob-archives "$DOMAIN-$DEV-*" \
+                    --glob-archives "'$DOMAIN-$DEV-*'" \
-                $PRUNE_OPTIONS
+                    "$PRUNE_OPTIONS"; then
-
-            if [ $? -ne 0 ]; then
                 # Commit any blocks.
                 blockCommit "$DOMSTATUS" "$DOMAIN" "$DEV"
                 fail "Failed to prune $DOMAIN ($DEV)"
@@ -198,46 +193,40 @@ while read -r line; do
 
     # Backup the domain info.
     echo "Backing up $DOMAIN xml"
-    virsh dumpxml "$DOMAIN" | borg create \
+    if ! virsh dumpxml "$DOMAIN" | borg create \
             --verbose \
             --stats \
             --show-rc \
-        "::$DOMAIN-xml-{now}" -
+            "::$DOMAIN-xml-{now}" -; then
-
-    if [ $? -ne 0 ]; then
         fail "Failed to backup $DOMAIN"
     fi
 
     # Prune if options are configured.
-    if [ -n "$PRUNE_OPTIONS" ]; then
+    if [[ -n "$PRUNE_OPTIONS" ]]; then
         echo "Pruning backups for $IMAGE"
-        borg prune --list \
+        if ! eval borg prune --list \
                 --show-rc \
-            --glob-archives "$DOMAIN-xml-*" \
+                --glob-archives "'$DOMAIN-xml-*'" \
-            $PRUNE_OPTIONS
+                "$PRUNE_OPTIONS"; then
-
-        if [ $? -ne 0 ]; then
             fail "Failed to prune $DOMAIN"
         fi
     fi
 done < <(
     virsh list --all | tail -n +3
-    echo ${PIPESTATUS[0]} >$DOMLIST_STATUS_TMP
+    echo "${PIPESTATUS[0]}" >"$DOMLIST_STATUS_TMP"
 )
 
 # Get status from the domain listing.
 status=1
-if [ -f $DOMLIST_STATUS_TMP ]; then
+if [[ -f $DOMLIST_STATUS_TMP ]]; then
-    status=$(cat $DOMLIST_STATUS_TMP)
+    status=$(cat "$DOMLIST_STATUS_TMP")
-    rm $DOMLIST_STATUS_TMP
+    rm "$DOMLIST_STATUS_TMP"
 fi
 
 # If status has an error, exit.
-if [ $status -ne 0 ]; then
+if ((status!=0)); then
     fail "Domain listing failed"
 fi
 
 # Shrink repo.
 borg compact
-
-rm "$PIDFILE"

kvm-backup-rbd.sh

@@ -7,7 +7,7 @@
 PIDFILE="/tmp/backup-image.pid"
 
 # If the pid file exists and process is running, exit.
-if [ -f "$PIDFILE" ]; then
+if [[ -f "$PIDFILE" ]]; then
     PID=$(cat "$PIDFILE")
     if ps -p "$PID" >/dev/null; then
         echo "Backup process already running, exiting."
@@ -18,10 +18,6 @@ fi
 # Create a new pid file for this process.
 echo $BASHPID >"$PIDFILE"
 
-# The pool in Ceph that you would like to backup.
-POOL="libvirt"
-# Pull images in pull from rbd driver.
-IMAGES=$(rbd -p $POOL ls)
 # The borg repository we're backing up to.
 export BORG_REPO='/media/Storage/Backup/kvm'
 # If you have a passphrase for your repository,
@@ -35,88 +31,194 @@ export BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO
 # Set to empty string to disable pruning.
 PRUNE_OPTIONS="--keep-daily 7 --keep-weekly 4 --keep-monthly 6"
 
+# Remove PID file on exit.
+cleanup() {
+    rm "$PIDFILE"
+}
+trap cleanup EXIT
+
+# Name the snapshot today's date.
+SNAPSHOT_NAME=$(date '+%Y-%m-%dT%H-%M-%S')
+
+# Keep number of snapshots in RBD.
+SNAPSHOTS_KEEP=0
+
+# Allows providing an argument of a domain to specifically backup.
+BACKUP_DOMAIN="$1"
+
 # Failures should remove pid file and exit with status code 1.
 fail() {
     echo "$1"
-    rm "$PIDFILE"
     exit 1
 }
 
-for IMAGE in $IMAGES; do
+# If the domain is running, commit the changes saved to the snapshot to the image to finish the backup.
+cleanupSnapshots() {
+    IMAGE="$1"
+    snapshots=()
+    
+    # Read list of snapshots for the provided image.
+    SNAPLIST_STATUS_TMP="/tmp/backup-snap-tmp"
+    while read -r _ NAME _; do
+        snapshots+=("$NAME")
+    done < <(
+        rbd snap list "$IMAGE" | tail -n +2
+        echo "${PIPESTATUS[0]}" >"$SNAPLIST_STATUS_TMP"
+    )
+
+    # Get status from the snapshot listing.
+    status=1
+    if [[ -f $SNAPLIST_STATUS_TMP ]]; then
+        status=$(cat "$SNAPLIST_STATUS_TMP")
+        rm "$SNAPLIST_STATUS_TMP"
+    fi
+
+    # If status has an error, exit.
+    if ((status!=0)); then
+        fail "Snapshot listing failed"
+    fi
+
+    # If the snapshot count is more than the number to keep,
+    # remove snapshots until count matches.
+    # The snapshots are listed from oldest to newest, so this
+    # should keep the newer snapshots.
+    snpashot_count=${#snapshots[@]}
+    if ((snpashot_count>=SNAPSHOTS_KEEP)); then
+        # Loop through snapshots until we removed enough to equal keep count.
+        for ((i = 0; snpashot_count-i > SNAPSHOTS_KEEP; i++)); do
+            NAME=${snapshots[$i]}
+            echo "Removing snapshot: $IMAGE@$NAME"
+            # Remove snapshot.
+            rbd snap remove "$IMAGE@$NAME"
+        done
+    fi
+}
+
+# I save the status in a temporary file so I can error out and exit if a failure occurs.
+DOMLIST_STATUS_TMP="/tmp/backup-image-domlist-tmp"
+while read -r _ DOMAIN _; do
+    # If the domain is empty, its not needed.
+    if [[ -z "$DOMAIN" ]]; then
+        continue
+    fi
+
+    # If a backup domain was provided, we're only going to backup that domain.
+    if [[ -n "$BACKUP_DOMAIN" ]] && [[ "$BACKUP_DOMAIN" != "$DOMAIN" ]]; then
+        continue
+    fi
+
+    # Get the images that need backing up.
+    DEVS=()
+    IMAGES=()
+    BLKLIST_STATUS_TMP="/tmp/backup-image-blklist-tmp"
+    while read -r DEV IMAGE; do
+        # Ignore empty line or no image.
+        if [[ -z "$IMAGE" ]] || [[ "$IMAGE" == "-" ]]; then
+            continue
+        fi
+
+        # Ignore iso files.
+        if [[ "$IMAGE" =~ \.iso$ ]]; then
+            continue
+        fi
+
+        # Ignore non-rbd files.
+        if [[ "$IMAGE" =~ ^\/ ]]; then
+            continue
+        fi
+
+        # This image needs backing up.
+        DEVS+=("$DEV")
+        IMAGES+=("$IMAGE")
+    done < <(
+        virsh domblklist "$DOMAIN" | tail -n +3
+        echo "${PIPESTATUS[0]}" >"$BLKLIST_STATUS_TMP"
+    )
+
+    # Get status from the block listing.
+    status=1
+    if [[ -f $BLKLIST_STATUS_TMP ]]; then
+        status=$(cat "$BLKLIST_STATUS_TMP")
+        rm "$BLKLIST_STATUS_TMP"
+    fi
+
+    # If status has an error, exit.
+    if ((status!=0)); then
+        fail "Domain block listing failed"
+    fi
+
+    # For each image we can backup, back it up.
+    for ((i = 0; i < ${#DEVS[@]}; i++)); do
+        DEV=${DEVS[$i]}
+        IMAGE=${IMAGES[$i]}
+        RBD_POOL=${IMAGE%/*}
+        RBD_IMAGE=${IMAGE##*/}
+        BACKUP_NAME="${RBD_POOL}_${RBD_IMAGE}"
+
+        # Create a snapshot.
+        rbd snap create "$IMAGE@$SNAPSHOT_NAME"
+
         # Export volume to borg backup.
         echo "Creating backup for $IMAGE"
-    rbd export $POOL/$IMAGE - | pv | borg create \
+        if ! rbd export "$IMAGE@$SNAPSHOT_NAME" - | pv | borg create \
                 --verbose \
                 --stats \
                 --show-rc \
-        "::$IMAGE-{now}" -
+                "::$BACKUP_NAME-{now}" -; then
-
-    if [ $? -ne 0 ]; then
             fail "Failed to backup $IMAGE"
         fi
 
         # Prune if options are configured.
-    if [ -n "$PRUNE_OPTIONS" ]; then
+        if [[ -n "$PRUNE_OPTIONS" ]]; then
             echo "Pruning backups for $IMAGE"
-        borg prune --list \
+            if ! eval borg prune --list \
                     --show-rc \
-            --glob-archives "$IMAGE-*" \
+                    --glob-archives "'$BACKUP_NAME-*'" \
-            $PRUNE_OPTIONS
+                    "$PRUNE_OPTIONS"; then
-
-        if [ $? -ne 0 ]; then
                 fail "Failed to prune $DOMAIN"
             fi
         fi
-done
 
-# I save the status in a temporary file so I can error out and exit if a failure occurs.
+       # Cleanup snapshots.
-DOMLIST_STATUS_TMP="/tmp/backup-rbd-domlist-tmp"
+       cleanupSnapshots "$IMAGE"
-while read -r line; do
+    done
-    # Extract the domain name from the line.
-    DOMAIN=$(echo $line | awk '{print $2}')
 
     # Backup the domain info.
     echo "Backing up $DOMAIN xml"
-    virsh dumpxml "$DOMAIN" | borg create \
+    if ! virsh dumpxml "$DOMAIN" | borg create \
             --verbose \
             --stats \
             --show-rc \
-        "::$DOMAIN-xml-{now}" -
+            "::$DOMAIN-xml-{now}" -; then
-
-    if [ $? -ne 0 ]; then
         fail "Failed to backup $DOMAIN"
     fi
 
     # Prune if options are configured.
-    if [ -n "$PRUNE_OPTIONS" ]; then
+    if [[ -n "$PRUNE_OPTIONS" ]]; then
         echo "Pruning backups for $IMAGE"
-        borg prune --list \
+        if ! eval borg prune --list \
                 --show-rc \
-            --glob-archives "$DOMAIN-xml-*" \
+                --glob-archives "'$DOMAIN-xml-*'" \
-            $PRUNE_OPTIONS
+                "$PRUNE_OPTIONS"; then
-
-        if [ $? -ne 0 ]; then
             fail "Failed to prune $DOMAIN"
         fi
     fi
 done < <(
     virsh list --all | tail -n +3
-    echo $? >$DOMLIST_STATUS_TMP
+    echo "${PIPESTATUS[0]}" >"$DOMLIST_STATUS_TMP"
 )
 
 # Get status from the domain listing.
 status=1
-if [ -f $DOMLIST_STATUS_TMP ]; then
+if [[ -f $DOMLIST_STATUS_TMP ]]; then
-    status=$(cat $DOMLIST_STATUS_TMP)
+    status=$(cat "$DOMLIST_STATUS_TMP")
-    rm $DOMLIST_STATUS_TMP
+    rm "$DOMLIST_STATUS_TMP"
 fi
 
 # If status has an error, exit.
-if [ $status -ne 0 ]; then
+if ((status!=0)); then
     fail "Domain listing failed"
 fi
 
 # Shrink repo.
 borg compact
-
-rm "$PIDFILE"