feat(rsync): check rpm repository metadata before and after stage 2

- Stage 2 published whatever metadata upstream served, so an upstream caught mid-update or serving a broken tree could publish a repomd.xml naming files it lacks, and clients failed with 404s.
- Before stage 2, list upstream repodata/ with the stage 2 filters applied and download each repomd.xml; skip stage 2 and fail the sync if any names a missing file or has a newest <timestamp> older than the published copy.
- When the check passes, set the mtime of any published repomd.xml whose content differs from upstream, and its signature, to 0 so rsync replaces it even when size and mtime match.
- After stage 2, verify every published repomd.xml against the files on disk and fail the sync, without running post_successful_sync_hook, if the tree is still torn.
- Add <module>_repodata_check, on by default for rpm modules with options_stage2 and rejected without it, and <module>_repodata_check_prune (default "Packages drpms isos images") to skip package directories in the walk.
- post_failed_sync now takes an optional reason that is printed and included in the failure email; the scratch directory is removed on SIGINT/SIGTERM.

README.md

@@ -251,6 +251,23 @@ This method supports an extra CLI argument, `--force`, which bypasses upstream c
 | `pre_stage2_hook` | no | *(none)* | A hook to run prior to the second stage sync. |
 | `post_hook` | no | *(none)* | A hook to call after a successful sync. If you are using jigdo, the hook is `jigdo_hook`. See [change counts in hooks](#change-counts-in-hooks). |
 | `jigdo_pkg_repo` | no | *(none)* | Path to the repo of packages, if you are using jigdo to build ISO images. |
+| `repodata_check` | no | `true` for an rpm `type` with `options_stage2`, else `false` | Check upstream's repository metadata before stage 2. See [checking rpm metadata before stage 2](#checking-rpm-metadata-before-stage-2). |
+| `repodata_check_prune` | no | `Packages drpms isos images` | Directory names the metadata check does not walk into, because they hold packages rather than metadata. |
+
+#### Checking rpm metadata before stage 2
+
+Stage 2 publishes whatever metadata upstream serves. An upstream caught mid-update, or one serving a broken tree, can publish a `repomd.xml` that names files it does not have, and every client of the mirror then fails with 404s. A stale upstream node can also serve an older generation of metadata than the one already published here.
+
+With `repodata_check` on, mirror-sync lists upstream's `repodata/` files and downloads each `repomd.xml` before stage 2. The stage 2 filters are applied, so only what stage 2 would publish is checked. Stage 2 is skipped and the sync counts as failed if any `repomd.xml`:
+
+- names a file upstream does not have, or
+- has a newest `<timestamp>` older than the copy already in `repo`.
+
+Stage 1 skips `repodata/*` and never deletes, so a skipped stage 2 leaves clients on the last consistent metadata. A failure that persists past `max_errors` emails `MAILTO`, as any failed sync does. If upstream rolled back on purpose, set `<module>_repodata_check="false"` for one sync to accept the older metadata.
+
+rsync treats a file as unchanged when its size and modification time match, and a `repomd.xml` regenerated upstream can match the old one on both. Stage 2 would then keep the old `repomd.xml` while deleting the files it names. When the check passes, any published `repomd.xml` whose content differs from upstream's, and its signature, gets a modification time of 0, so stage 2 replaces it.
+
+After stage 2, every published `repomd.xml` is checked against the files on disk. A tree that is still torn, for example because upstream changed between the check and stage 2, fails the sync instead of being recorded as a success, and no `post_successful_sync_hook` runs.
 
 #### Examples
 

mirror-sync.sh

@@ -6,7 +6,7 @@ PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:$HOME/.local/
 
 # Variables for trace generation.
 PROGRAM="mirror-sync"
-VERSION="20260727"
+VERSION="20261017"
 TRACEHOST=$(hostname -f)
 mirror_hostname=$(hostname -f)
 DATE_STARTED=$(LC_ALL=POSIX LANG=POSIX date -u -R)
@@ -77,6 +77,9 @@ fi
 # temporary stage logs, ...). The sync functions append to this list as they
 # create such files so that a Ctrl-C does not leave stale state behind.
 cleanup_files=()
+# Directories to remove if a sync is interrupted, such as the scratch space the
+# repository metadata check downloads into.
+cleanup_dirs=()
 
 # Handle Ctrl-C (SIGINT) and termination (SIGTERM). When run from a terminal
 # the signal is delivered to the whole process group, so the rsync/aws/git/...
@@ -96,6 +99,9 @@ handle_interrupt() {
     for _f in "${cleanup_files[@]}"; do
         [[ $_f ]] && rm -f "$_f"
     done
+    for _f in "${cleanup_dirs[@]}"; do
+        [[ $_f ]] && rm -rf "$_f"
+    done
 
     # Remove the pid lock file for this module so a later run is not blocked.
     [[ $PIDFILE ]] && rm -f "$PIDFILE"
@@ -813,8 +819,12 @@ run_post_successful_sync_hook() {
     fi
 }
 
-# On failed sync, count failure and exit.
+# On failed sync, count failure and exit. An optional reason is printed and
+# added to the email, for failures the log alone would not make obvious.
 post_failed_sync() {
+    if [[ $1 ]]; then
+        echo "$1"
+    fi
     echo "Sync failed."
     
     # Update failure count.
@@ -822,7 +832,7 @@ post_failed_sync() {
 
     # If failure count is over our maximum defined errors, email about the failure.
     if ((new_error_count>max_errors)); then
-        mail_error "Unable to sync with $MODULE, check logs."
+        mail_error "Unable to sync with $MODULE, check logs.${1:+ $1}"
         
         # Remove the error count file so that the count resets.
         rm -f "$ERRORFILE"
@@ -1523,6 +1533,202 @@ read_rsync_config() {
     eval repo_type="\$${MODULE}_type"
 }
 
+# Read whether this module's upstream metadata is checked before stage 2. It
+# defaults to on for rpm modules synced in two stages, since stage 2 is where
+# their metadata is published, and it means nothing without a second stage.
+read_repodata_check_config() {
+    local _default="false"
+    if [[ $repo_type == "rpm" && $options_stage2 ]]; then
+        _default="true"
+    fi
+    eval repodata_check_enabled="\$${MODULE}_repodata_check"
+    parse_bool repodata_check_enabled "$_default" "${MODULE}_repodata_check"
+    if [[ $repodata_check_enabled && ! $options_stage2 ]]; then
+        echo "${MODULE}_repodata_check needs ${MODULE}_options_stage2, the check runs before stage 2."
+        exit 1
+    fi
+    eval repodata_check_prune="\$${MODULE}_repodata_check_prune"
+    repodata_check_prune="${repodata_check_prune:-Packages drpms isos images}"
+}
+
+# Print the newest <timestamp> in a repomd.xml, or 0 when it has none.
+repomd_timestamp() {
+    local _ts
+    _ts=$(grep -o '<timestamp>[0-9]*' "$1" 2>/dev/null | sed 's/<timestamp>//' | sort -n | tail -n 1)
+    echo "${_ts:-0}"
+}
+
+# Print the locations a repomd.xml names that are relative to its repository.
+# Locations under an xml:base or with an absolute path are served from
+# elsewhere and are left out.
+repomd_locations() {
+    local _location _href
+    while IFS= read -r _location; do
+        if [[ $_location == *xml:base* ]]; then
+            continue
+        fi
+        _href=${_location#*href=\"}
+        _href=${_href%%\"*}
+        if [[ $_href == /* || $_href == *://* ]]; then
+            continue
+        fi
+        echo "$_href"
+    done < <(grep -o '<location [^>]*>' "$1")
+}
+
+# Check the repository metadata upstream would publish in stage 2, before
+# stage 2 runs. Every file a repomd.xml names must exist upstream, and no
+# repomd.xml may be older than the copy already published here. An upstream
+# caught mid-update, serving a torn tree, or rolled back to an older generation
+# fails the check. Stage 1 skips repodata and never deletes, so refusing stage
+# 2 keeps the mirror on its last consistent metadata. Returns non-zero, with
+# the problems printed, when stage 2 must not run.
+#
+# A published repomd.xml can differ from upstream's while matching it in size
+# and modification time, which rsync takes to mean the file is unchanged, so
+# stage 2 would leave it naming files that are gone. When the check passes,
+# such a repomd.xml and its signature are given a modification time of 0 so
+# stage 2 replaces them.
+check_rpm_repodata() {
+    local _tmp _prune _dir _line _path _repomd _base _href _upstream_ts _local_ts _f
+    local _checked=0 _torn=0 _older=0
+    local -a _stale=()
+    local -A _upstream_files=()
+
+    _tmp=$(mktemp -d "${tmpDirBase:?}/${MODULE}-repodata.XXXXXX") || {
+        echo "Could not create a scratch directory under ${tmpDirBase}."
+        return 1
+    }
+    cleanup_dirs+=("$_tmp")
+
+    # Directories that hold packages rather than metadata are left out of the
+    # walk to keep the listing fast.
+    _prune=""
+    for _dir in ${repodata_check_prune}; do
+        _prune+=" --exclude '${_dir}/'"
+    done
+
+    # List every metadata file upstream. The stage 2 options come first, so
+    # their filters limit the check to what stage 2 would publish.
+    if ! eval "$sync_timeout" rsync -rl \
+            --no-motd           \
+            --list-only         \
+            --no-human-readable \
+            "${options_stage2:-}" \
+            "$_prune"           \
+            --include "'*/'"    \
+            --include "'repodata/*'" \
+            --exclude "'*'"     \
+            "'${source:?}'" > "$_tmp/listing"; then
+        echo "Could not list the upstream repository metadata."
+        rm -rf "$_tmp"
+        return 1
+    fi
+    while IFS= read -r _line; do
+        if [[ $_line =~ ^[-l][^\ ]*\ +[0-9]+\ [0-9/]+\ [0-9:]+\ (.*)$ ]]; then
+            _path=${BASH_REMATCH[1]}
+            _upstream_files["${_path%% -> *}"]=1
+        fi
+    done < "$_tmp/listing"
+
+    # Download only the repomd.xml files, by name, so upstream is not walked a
+    # second time.
+    for _path in "${!_upstream_files[@]}"; do
+        if [[ $_path == repodata/repomd.xml || $_path == */repodata/repomd.xml ]]; then
+            echo "$_path"
+        fi
+    done | sort > "$_tmp/repomds"
+    if [[ ! -s $_tmp/repomds ]]; then
+        echo "Upstream lists no repodata/repomd.xml, nothing to check."
+        rm -rf "$_tmp"
+        return 0
+    fi
+    mkdir "$_tmp/upstream"
+    if ! $sync_timeout rsync \
+            --no-motd \
+            --copy-links \
+            --files-from="$_tmp/repomds" \
+            "${source:?}" "$_tmp/upstream/"; then
+        echo "Could not download the upstream repomd.xml files."
+        rm -rf "$_tmp"
+        return 1
+    fi
+
+    while IFS= read -r _repomd; do
+        _checked=$((_checked+1))
+        _base=${_repomd%repodata/repomd.xml}
+
+        # Every relative location must exist upstream.
+        while IFS= read -r _href; do
+            if [[ ! ${_upstream_files["${_base}${_href}"]} ]]; then
+                echo "Torn: ${_repomd} names ${_href}, which upstream does not have."
+                _torn=$((_torn+1))
+                break
+            fi
+        done < <(repomd_locations "$_tmp/upstream/$_repomd")
+
+        # The metadata must be no older than what is published here.
+        if [[ -f ${repo%/}/$_repomd ]]; then
+            _upstream_ts=$(repomd_timestamp "$_tmp/upstream/$_repomd")
+            _local_ts=$(repomd_timestamp "${repo%/}/$_repomd")
+            if (( _upstream_ts > 0 && _local_ts > 0 && _upstream_ts < _local_ts )); then
+                echo "Older: ${_repomd} is from $(date -u -d "@${_upstream_ts}" '+%F %T') UTC, ours is from $(date -u -d "@${_local_ts}" '+%F %T') UTC."
+                _older=$((_older+1))
+            fi
+            if ! cmp -s "$_tmp/upstream/$_repomd" "${repo%/}/$_repomd"; then
+                _stale+=("${repo%/}/$_repomd")
+            fi
+        fi
+    done < "$_tmp/repomds"
+    rm -rf "$_tmp"
+
+    echo "Checked ${_checked} repomd.xml: ${_torn} torn, ${_older} older than ours, ${#_stale[@]} changed."
+    if (( _torn > 0 || _older > 0 )); then
+        return 1
+    fi
+
+    # Make sure stage 2 replaces every repomd.xml whose content changed.
+    for _repomd in "${_stale[@]}"; do
+        for _f in "$_repomd" "$_repomd".*; do
+            if [[ -f $_f && ! -L $_f ]]; then
+                touch -d @0 "$_f"
+            fi
+        done
+    done
+    return 0
+}
+
+# Check the repository metadata published here after stage 2: every file a
+# repomd.xml names must exist. This catches a torn tree however it came about,
+# including one left by an earlier sync, so it is reported instead of being
+# recorded as a successful sync. Returns non-zero, with the problems printed,
+# when the published tree is torn.
+verify_published_repodata() {
+    local _dir _repomd _base _href _checked=0 _torn=0
+    local -a _prune=()
+
+    for _dir in ${repodata_check_prune}; do
+        _prune+=(-name "$_dir" -o)
+    done
+    while IFS= read -r _repomd; do
+        _checked=$((_checked+1))
+        _base=${_repomd%repodata/repomd.xml}
+        while IFS= read -r _href; do
+            if [[ ! -e ${_base}${_href} ]]; then
+                echo "Torn: ${_repomd#"${repo%/}/"} names ${_href}, which is missing here."
+                _torn=$((_torn+1))
+                break
+            fi
+        done < <(repomd_locations "$_repomd")
+    done < <(find "${repo%/}/" \( "${_prune[@]}" -false \) -prune -o -path '*/repodata/repomd.xml' -type f -print)
+
+    echo "Checked ${_checked} published repomd.xml: ${_torn} torn."
+    if (( _torn > 0 )); then
+        return 1
+    fi
+    return 0
+}
+
 # Modules based on rsync.
 rsync_sync() {
     # Start the module.
@@ -1553,6 +1759,7 @@ rsync_sync() {
     eval pre_stage2_hook="\$${MODULE}_pre_stage2_hook"
     eval upstream_check="\$${MODULE}_upstream_check"
     eval time_file_check="\$${MODULE}_time_file_check"
+    read_repodata_check_config
 
     # Check if upstream was updated recently if configured.
     # This is designed to slow down rsync so we only rsync
@@ -1688,6 +1895,16 @@ rsync_sync() {
             esac
         done
 
+        # Refuse to publish upstream metadata that is torn or older than ours.
+        if [[ $repodata_check_enabled ]]; then
+            echo
+            echo "Checking upstream repository metadata:"
+            if ! check_rpm_repodata; then
+                rm -f "$mirror_update_file"
+                post_failed_sync "Upstream repository metadata failed its check, so stage 2 was skipped and the published metadata was kept."
+            fi
+        fi
+
         # Run any hooks.
         if [[ $pre_stage2_hook ]]; then
             echo "Executing pre-stage2 hook:"
@@ -1726,6 +1943,16 @@ rsync_sync() {
             rm -f "$mirror_update_file"
             post_failed_sync
         fi
+
+        # Confirm the metadata stage 2 left published is whole.
+        if [[ $repodata_check_enabled ]]; then
+            echo
+            echo "Checking published repository metadata:"
+            if ! verify_published_repodata; then
+                rm -f "$mirror_update_file"
+                post_failed_sync "The published repository metadata is torn after stage 2."
+            fi
+        fi
     fi
     
     # Count what the sync changed, before the logs it is counted from are