GRMrGecko/kvm-backup-scripts
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update rbd to work with snapshots and virsh domblklist. Refactor images backup to newer code style.

Browse Source
This commit is contained in:
James Coleman 2024-04-19 08:45:01 -05:00
parent 9d699a27ef
commit 8362d93969
2 changed files with 218 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
kvm-backup-images.sh Normal file → Executable file
View File

@ -12,7 +12,7 @@
PIDFILE="/tmp/backup-image.pid" PIDFILE="/tmp/backup-image.pid"
# If the pid file exists and process is running, exit. # If the pid file exists and process is running, exit.
if [ -f "$PIDFILE" ]; then if [[ -f "$PIDFILE" ]]; then
PID=$(cat "$PIDFILE") PID=$(cat "$PIDFILE")
if ps -p "$PID" >/dev/null; then if ps -p "$PID" >/dev/null; then
echo "Backup process already running, exiting." echo "Backup process already running, exiting."
@ -36,13 +36,18 @@ export BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO
# Set to empty string to disable pruning. # Set to empty string to disable pruning.
PRUNE_OPTIONS="--keep-daily 7 --keep-weekly 4 --keep-monthly 6" PRUNE_OPTIONS="--keep-daily 7 --keep-weekly 4 --keep-monthly 6"
# Remove PID file on exit.
cleanup() {
rm "$PIDFILE"
}
trap cleanup EXIT
# Allows providing an argument of a domain to specifically backup. # Allows providing an argument of a domain to specifically backup.
BACKUP_DOMAIN="$1" BACKUP_DOMAIN="$1"
# Failures should remove pid file and exit with status code 1. # Failures should remove pid file and exit with status code 1.
fail() { fail() {
echo "$1" echo "$1"
rm "$PIDFILE"
exit 1 exit 1
} }
@ -53,15 +58,14 @@ blockCommit() {
DEV="$3" DEV="$3"
if [[ "$DOMSTATUS" == "running" ]]; then if [[ "$DOMSTATUS" == "running" ]]; then
echo "Commit changes for $DOMAIN ($DEV)" echo "Commit changes for $DOMAIN ($DEV)"
virsh blockcommit \
"$DOMAIN" \
"$DEV" \
--active \
--verbose \
--pivot \
--delete
if [ $? -ne 0 ]; then if ! virsh blockcommit \
"$DOMAIN" \
"$DEV" \
--active \
--verbose \
--pivot \
--delete; then
fail "Could not commit changes $DOMAIN ($DEV). This may be a major issue and VM may be broken now." fail "Could not commit changes $DOMAIN ($DEV). This may be a major issue and VM may be broken now."
fi fi
fi fi
@ -69,18 +73,14 @@ blockCommit() {
# I save the status in a temporary file so I can error out and exit if a failure occurs. # I save the status in a temporary file so I can error out and exit if a failure occurs.
DOMLIST_STATUS_TMP="/tmp/backup-image-domlist-tmp" DOMLIST_STATUS_TMP="/tmp/backup-image-domlist-tmp"
while read -r line; do while read -r _ DOMAIN DOMSTATUS; do
# Extract the domain name and status from the line.
DOMAIN=$(echo $line | awk '{print $2}')
DOMSTATUS=$(echo $line | awk '{for (i=3; i<NF; i++) printf $i " "; if (NF>=3) print $NF}')
# If the domain is empty, its not needed. # If the domain is empty, its not needed.
if [ -z "$DOMAIN" ]; then if [[ -z "$DOMAIN" ]]; then
continue continue
fi fi
# If a backup domain was provided, we're only going to backup that domain. # If a backup domain was provided, we're only going to backup that domain.
if [ -n "$BACKUP_DOMAIN" ] && [[ "$BACKUP_DOMAIN" != "$DOMAIN" ]]; then if [[ -n "$BACKUP_DOMAIN" ]] && [[ "$BACKUP_DOMAIN" != "$DOMAIN" ]]; then
continue continue
fi fi
@ -88,13 +88,9 @@ while read -r line; do
DEVS=() DEVS=()
IMAGES=() IMAGES=()
BLKLIST_STATUS_TMP="/tmp/backup-image-blklist-tmp" BLKLIST_STATUS_TMP="/tmp/backup-image-blklist-tmp"
while read -r line; do while read -r DEV IMAGE; do
# Extract the device and image from the line.
DEV=$(echo $line | awk '{print $1}')
IMAGE=$(echo $line | awk '{for (i=2; i<NF; i++) printf $i " "; if (NF>=2) print $NF}')
# Ignore empty line or no image. # Ignore empty line or no image.
if [ -z "$IMAGE" ] || [[ "$IMAGE" == "-" ]]; then if [[ -z "$IMAGE" ]] || [[ "$IMAGE" == "-" ]]; then
continue continue
fi fi
@ -103,23 +99,28 @@ while read -r line; do
continue continue
fi fi
# Ignore non-image files.
if ! [[ "$IMAGE" =~ ^\/ ]]; then
continue
fi
# This image needs backing up. # This image needs backing up.
DEVS+=("$DEV") DEVS+=("$DEV")
IMAGES+=("$IMAGE") IMAGES+=("$IMAGE")
done < <( done < <(
virsh domblklist $DOMAIN | tail -n +3 virsh domblklist "$DOMAIN" | tail -n +3
echo ${PIPESTATUS[0]} >$BLKLIST_STATUS_TMP echo "${PIPESTATUS[0]}" >"$BLKLIST_STATUS_TMP"
) )
# Get status from the block listing. # Get status from the block listing.
status=1 status=1
if [ -f $BLKLIST_STATUS_TMP ]; then if [[ -f $BLKLIST_STATUS_TMP ]]; then
status=$(cat $BLKLIST_STATUS_TMP) status=$(cat "$BLKLIST_STATUS_TMP")
rm $BLKLIST_STATUS_TMP rm "$BLKLIST_STATUS_TMP"
fi fi
# If status has an error, exit. # If status has an error, exit.
if [ $status -ne 0 ]; then if ((status!=0)); then
fail "Domain block listing failed" fail "Domain block listing failed"
fi fi
@ -135,7 +136,7 @@ while read -r line; do
if [[ "$DOMSTATUS" == "running" ]]; then if [[ "$DOMSTATUS" == "running" ]]; then
# If the snapshot file exists, we should commit changes before performing another snapshot. # If the snapshot file exists, we should commit changes before performing another snapshot.
# We are assuming that we created the snapshot here, and that concurrent runs are not possible. # We are assuming that we created the snapshot here, and that concurrent runs are not possible.
if [ -e "$IMAGESNAPSHOT" ]; then if [[ -e "$IMAGESNAPSHOT" ]]; then
# Commit any blocks. # Commit any blocks.
blockCommit "$DOMSTATUS" "$DOMAIN" "$DEV" blockCommit "$DOMSTATUS" "$DOMAIN" "$DEV"
fi fi
@ -150,42 +151,36 @@ while read -r line; do
fi fi
echo "Creating snapshot for $DOMAIN ($DEV)" echo "Creating snapshot for $DOMAIN ($DEV)"
virsh snapshot-create-as --domain "$DOMAIN" \ if ! virsh snapshot-create-as --domain "$DOMAIN" \
--name backup \ --name backup \
--no-metadata \ --no-metadata \
--atomic \ --atomic \
--disk-only \ --disk-only \
--diskspec $DEV,snapshot=external --diskspec "$DEV,snapshot=external"; then
if [ $? -ne 0 ]; then
fail "Failed to create snapshot for $DOMAIN ($DEV)" fail "Failed to create snapshot for $DOMAIN ($DEV)"
fi fi
fi fi
# Backup the image. # Backup the image.
echo "Creating backup for $DOMAIN ($DEV [$IMAGE])" echo "Creating backup for $DOMAIN ($DEV [$IMAGE])"
pv "$IMAGE" | borg create \ if ! pv "$IMAGE" | borg create \
--verbose \ --verbose \
--stats \ --stats \
--show-rc \ --show-rc \
--stdin-name "$IMAGENAME" \ --stdin-name "$IMAGENAME" \
"::$DOMAIN-$DEV-{now}" - "::$DOMAIN-$DEV-{now}" -; then
if [ $? -ne 0 ]; then
# Commit any blocks. # Commit any blocks.
blockCommit "$DOMSTATUS" "$DOMAIN" "$DEV" blockCommit "$DOMSTATUS" "$DOMAIN" "$DEV"
fail "Failed to backup $DOMAIN ($DEV)" fail "Failed to backup $DOMAIN ($DEV)"
fi fi
# Prune if options are configured. # Prune if options are configured.
if [ -n "$PRUNE_OPTIONS" ]; then if [[ -n "$PRUNE_OPTIONS" ]]; then
echo "Pruning backups for $DOMAIN ($DEV)" echo "Pruning backups for $DOMAIN ($DEV)"
borg prune --list \ if ! eval borg prune --list \
--show-rc \ --show-rc \
--glob-archives "$DOMAIN-$DEV-*" \ --glob-archives "'$DOMAIN-$DEV-*'" \
$PRUNE_OPTIONS "$PRUNE_OPTIONS"; then
if [ $? -ne 0 ]; then
# Commit any blocks. # Commit any blocks.
blockCommit "$DOMSTATUS" "$DOMAIN" "$DEV" blockCommit "$DOMSTATUS" "$DOMAIN" "$DEV"
fail "Failed to prune $DOMAIN ($DEV)" fail "Failed to prune $DOMAIN ($DEV)"
@ -198,46 +193,40 @@ while read -r line; do
# Backup the domain info. # Backup the domain info.
echo "Backing up $DOMAIN xml" echo "Backing up $DOMAIN xml"
virsh dumpxml "$DOMAIN" | borg create \ if ! virsh dumpxml "$DOMAIN" | borg create \
--verbose \ --verbose \
--stats \ --stats \
--show-rc \ --show-rc \
"::$DOMAIN-xml-{now}" - "::$DOMAIN-xml-{now}" -; then
if [ $? -ne 0 ]; then
fail "Failed to backup $DOMAIN" fail "Failed to backup $DOMAIN"
fi fi
# Prune if options are configured. # Prune if options are configured.
if [ -n "$PRUNE_OPTIONS" ]; then if [[ -n "$PRUNE_OPTIONS" ]]; then
echo "Pruning backups for $IMAGE" echo "Pruning backups for $IMAGE"
borg prune --list \ if ! eval borg prune --list \
--show-rc \ --show-rc \
--glob-archives "$DOMAIN-xml-*" \ --glob-archives "'$DOMAIN-xml-*'" \
$PRUNE_OPTIONS "$PRUNE_OPTIONS"; then
if [ $? -ne 0 ]; then
fail "Failed to prune $DOMAIN" fail "Failed to prune $DOMAIN"
fi fi
fi fi
done < <( done < <(
virsh list --all | tail -n +3 virsh list --all | tail -n +3
echo ${PIPESTATUS[0]} >$DOMLIST_STATUS_TMP echo "${PIPESTATUS[0]}" >"$DOMLIST_STATUS_TMP"
) )
# Get status from the domain listing. # Get status from the domain listing.
status=1 status=1
if [ -f $DOMLIST_STATUS_TMP ]; then if [[ -f $DOMLIST_STATUS_TMP ]]; then
status=$(cat $DOMLIST_STATUS_TMP) status=$(cat "$DOMLIST_STATUS_TMP")
rm $DOMLIST_STATUS_TMP rm "$DOMLIST_STATUS_TMP"
fi fi
# If status has an error, exit. # If status has an error, exit.
if [ $status -ne 0 ]; then if ((status!=0)); then
fail "Domain listing failed" fail "Domain listing failed"
fi fi
# Shrink repo. # Shrink repo.
borg compact borg compact
rm "$PIDFILE"

208
kvm-backup-rbd.sh Normal file → Executable file
View File

@ -7,7 +7,7 @@
PIDFILE="/tmp/backup-image.pid" PIDFILE="/tmp/backup-image.pid"
# If the pid file exists and process is running, exit. # If the pid file exists and process is running, exit.
if [ -f "$PIDFILE" ]; then if [[ -f "$PIDFILE" ]]; then
PID=$(cat "$PIDFILE") PID=$(cat "$PIDFILE")
if ps -p "$PID" >/dev/null; then if ps -p "$PID" >/dev/null; then
echo "Backup process already running, exiting." echo "Backup process already running, exiting."
@ -18,10 +18,6 @@ fi
# Create a new pid file for this process. # Create a new pid file for this process.
echo $BASHPID >"$PIDFILE" echo $BASHPID >"$PIDFILE"
# The pool in Ceph that you would like to backup.
POOL="libvirt"
# Pull images in pull from rbd driver.
IMAGES=$(rbd -p $POOL ls)
# The borg repository we're backing up to. # The borg repository we're backing up to.
export BORG_REPO='/media/Storage/Backup/kvm' export BORG_REPO='/media/Storage/Backup/kvm'
# If you have a passphrase for your repository, # If you have a passphrase for your repository,
@ -35,88 +31,194 @@ export BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO
# Set to empty string to disable pruning. # Set to empty string to disable pruning.
PRUNE_OPTIONS="--keep-daily 7 --keep-weekly 4 --keep-monthly 6" PRUNE_OPTIONS="--keep-daily 7 --keep-weekly 4 --keep-monthly 6"
# Remove PID file on exit.
cleanup() {
rm "$PIDFILE"
}
trap cleanup EXIT
# Name the snapshot today's date.
SNAPSHOT_NAME=$(date '+%Y-%m-%dT%H-%M-%S')
# Keep number of snapshots in RBD.
SNAPSHOTS_KEEP=0
# Allows providing an argument of a domain to specifically backup.
BACKUP_DOMAIN="$1"
# Failures should remove pid file and exit with status code 1. # Failures should remove pid file and exit with status code 1.
fail() { fail() {
echo "$1" echo "$1"
rm "$PIDFILE"
exit 1 exit 1
} }
for IMAGE in $IMAGES; do # If the domain is running, commit the changes saved to the snapshot to the image to finish the backup.
# Export volume to borg backup. cleanupSnapshots() {
echo "Creating backup for $IMAGE" IMAGE="$1"
rbd export $POOL/$IMAGE - | pv | borg create \ snapshots=()
--verbose \
--stats \
--show-rc \
"::$IMAGE-{now}" -
if [ $? -ne 0 ]; then # Read list of snapshots for the provided image.
fail "Failed to backup $IMAGE" SNAPLIST_STATUS_TMP="/tmp/backup-snap-tmp"
while read -r _ NAME _; do
snapshots+=("$NAME")
done < <(
rbd snap list "$IMAGE" | tail -n +2
echo "${PIPESTATUS[0]}" >"$SNAPLIST_STATUS_TMP"
)
# Get status from the snapshot listing.
status=1
if [[ -f $SNAPLIST_STATUS_TMP ]]; then
status=$(cat "$SNAPLIST_STATUS_TMP")
rm "$SNAPLIST_STATUS_TMP"
fi fi
# Prune if options are configured. # If status has an error, exit.
if [ -n "$PRUNE_OPTIONS" ]; then if ((status!=0)); then
echo "Pruning backups for $IMAGE" fail "Snapshot listing failed"
borg prune --list \
--show-rc \
--glob-archives "$IMAGE-*" \
$PRUNE_OPTIONS
if [ $? -ne 0 ]; then
fail "Failed to prune $DOMAIN"
fi
fi fi
done
# If the snapshot count is more than the number to keep,
# remove snapshots until count matches.
# The snapshots are listed from oldest to newest, so this
# should keep the newer snapshots.
snpashot_count=${#snapshots[@]}
if ((snpashot_count>=SNAPSHOTS_KEEP)); then
# Loop through snapshots until we removed enough to equal keep count.
for ((i = 0; snpashot_count-i > SNAPSHOTS_KEEP; i++)); do
NAME=${snapshots[$i]}
echo "Removing snapshot: $IMAGE@$NAME"
# Remove snapshot.
rbd snap remove "$IMAGE@$NAME"
done
fi
}
# I save the status in a temporary file so I can error out and exit if a failure occurs. # I save the status in a temporary file so I can error out and exit if a failure occurs.
DOMLIST_STATUS_TMP="/tmp/backup-rbd-domlist-tmp" DOMLIST_STATUS_TMP="/tmp/backup-image-domlist-tmp"
while read -r line; do while read -r _ DOMAIN _; do
# Extract the domain name from the line. # If the domain is empty, its not needed.
DOMAIN=$(echo $line | awk '{print $2}') if [[ -z "$DOMAIN" ]]; then
continue
fi
# If a backup domain was provided, we're only going to backup that domain.
if [[ -n "$BACKUP_DOMAIN" ]] && [[ "$BACKUP_DOMAIN" != "$DOMAIN" ]]; then
continue
fi
# Get the images that need backing up.
DEVS=()
IMAGES=()
BLKLIST_STATUS_TMP="/tmp/backup-image-blklist-tmp"
while read -r DEV IMAGE; do
# Ignore empty line or no image.
if [[ -z "$IMAGE" ]] || [[ "$IMAGE" == "-" ]]; then
continue
fi
# Ignore iso files.
if [[ "$IMAGE" =~ \.iso$ ]]; then
continue
fi
# Ignore non-rbd files.
if [[ "$IMAGE" =~ ^\/ ]]; then
continue
fi
# This image needs backing up.
DEVS+=("$DEV")
IMAGES+=("$IMAGE")
done < <(
virsh domblklist "$DOMAIN" | tail -n +3
echo "${PIPESTATUS[0]}" >"$BLKLIST_STATUS_TMP"
)
# Get status from the block listing.
status=1
if [[ -f $BLKLIST_STATUS_TMP ]]; then
status=$(cat "$BLKLIST_STATUS_TMP")
rm "$BLKLIST_STATUS_TMP"
fi
# If status has an error, exit.
if ((status!=0)); then
fail "Domain block listing failed"
fi
# For each image we can backup, back it up.
for ((i = 0; i < ${#DEVS[@]}; i++)); do
DEV=${DEVS[$i]}
IMAGE=${IMAGES[$i]}
RBD_POOL=${IMAGE%/*}
RBD_IMAGE=${IMAGE##*/}
BACKUP_NAME="${RBD_POOL}_${RBD_IMAGE}"
# Create a snapshot.
rbd snap create "$IMAGE@$SNAPSHOT_NAME"
# Export volume to borg backup.
echo "Creating backup for $IMAGE"
if ! rbd export "$IMAGE@$SNAPSHOT_NAME" - | pv | borg create \
--verbose \
--stats \
--show-rc \
"::$BACKUP_NAME-{now}" -; then
fail "Failed to backup $IMAGE"
fi
# Prune if options are configured.
if [[ -n "$PRUNE_OPTIONS" ]]; then
echo "Pruning backups for $IMAGE"
if ! eval borg prune --list \
--show-rc \
--glob-archives "'$BACKUP_NAME-*'" \
"$PRUNE_OPTIONS"; then
fail "Failed to prune $DOMAIN"
fi
fi
# Cleanup snapshots.
cleanupSnapshots "$IMAGE"
done
# Backup the domain info. # Backup the domain info.
echo "Backing up $DOMAIN xml" echo "Backing up $DOMAIN xml"
virsh dumpxml "$DOMAIN" | borg create \ if ! virsh dumpxml "$DOMAIN" | borg create \
--verbose \ --verbose \
--stats \ --stats \
--show-rc \ --show-rc \
"::$DOMAIN-xml-{now}" - "::$DOMAIN-xml-{now}" -; then
if [ $? -ne 0 ]; then
fail "Failed to backup $DOMAIN" fail "Failed to backup $DOMAIN"
fi fi
# Prune if options are configured. # Prune if options are configured.
if [ -n "$PRUNE_OPTIONS" ]; then if [[ -n "$PRUNE_OPTIONS" ]]; then
echo "Pruning backups for $IMAGE" echo "Pruning backups for $IMAGE"
borg prune --list \ if ! eval borg prune --list \
--show-rc \ --show-rc \
--glob-archives "$DOMAIN-xml-*" \ --glob-archives "'$DOMAIN-xml-*'" \
$PRUNE_OPTIONS "$PRUNE_OPTIONS"; then
if [ $? -ne 0 ]; then
fail "Failed to prune $DOMAIN" fail "Failed to prune $DOMAIN"
fi fi
fi fi
done < <( done < <(
virsh list --all | tail -n +3 virsh list --all | tail -n +3
echo $? >$DOMLIST_STATUS_TMP echo "${PIPESTATUS[0]}" >"$DOMLIST_STATUS_TMP"
) )
# Get status from the domain listing. # Get status from the domain listing.
status=1 status=1
if [ -f $DOMLIST_STATUS_TMP ]; then if [[ -f $DOMLIST_STATUS_TMP ]]; then
status=$(cat $DOMLIST_STATUS_TMP) status=$(cat "$DOMLIST_STATUS_TMP")
rm $DOMLIST_STATUS_TMP rm "$DOMLIST_STATUS_TMP"
fi fi
# If status has an error, exit. # If status has an error, exit.
if [ $status -ne 0 ]; then if ((status!=0)); then
fail "Domain listing failed" fail "Domain listing failed"
fi fi
# Shrink repo. # Shrink repo.
borg compact borg compact
rm "$PIDFILE"