diff --git a/README.md b/README.md index a771337..b2309b5 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,23 @@ This method supports an extra CLI argument, `--force`, which bypasses upstream c | `pre_stage2_hook` | no | *(none)* | A hook to run prior to the second stage sync. | | `post_hook` | no | *(none)* | A hook to call after a successful sync. If you are using jigdo, the hook is `jigdo_hook`. See [change counts in hooks](#change-counts-in-hooks). | | `jigdo_pkg_repo` | no | *(none)* | Path to the repo of packages, if you are using jigdo to build ISO images. | +| `repodata_check` | no | `true` for an rpm `type` with `options_stage2`, else `false` | Check upstream's repository metadata before stage 2. See [checking rpm metadata before stage 2](#checking-rpm-metadata-before-stage-2). | +| `repodata_check_prune` | no | `Packages drpms isos images` | Directory names the metadata check does not walk into, because they hold packages rather than metadata. | + +#### Checking rpm metadata before stage 2 + +Stage 2 publishes whatever metadata upstream serves. An upstream caught mid-update, or one serving a broken tree, can publish a `repomd.xml` that names files it does not have, and every client of the mirror then fails with 404s. A stale upstream node can also serve an older generation of metadata than the one already published here. + +With `repodata_check` on, mirror-sync lists upstream's `repodata/` files and downloads each `repomd.xml` before stage 2. The stage 2 filters are applied, so only what stage 2 would publish is checked. Stage 2 is skipped and the sync counts as failed if any `repomd.xml`: + +- names a file upstream does not have, or +- has a newest `` older than the copy already in `repo`. + +Stage 1 skips `repodata/*` and never deletes, so a skipped stage 2 leaves clients on the last consistent metadata. A failure that persists past `max_errors` emails `MAILTO`, as any failed sync does. If upstream rolled back on purpose, set `_repodata_check="false"` for one sync to accept the older metadata. + +rsync treats a file as unchanged when its size and modification time match, and a `repomd.xml` regenerated upstream can match the old one on both. Stage 2 would then keep the old `repomd.xml` while deleting the files it names. When the check passes, any published `repomd.xml` whose content differs from upstream's, and its signature, gets a modification time of 0, so stage 2 replaces it. + +After stage 2, every published `repomd.xml` is checked against the files on disk. A tree that is still torn, for example because upstream changed between the check and stage 2, fails the sync instead of being recorded as a success, and no `post_successful_sync_hook` runs. #### Examples diff --git a/mirror-sync.sh b/mirror-sync.sh index fe8d568..07919f1 100644 --- a/mirror-sync.sh +++ b/mirror-sync.sh @@ -6,7 +6,7 @@ PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:$HOME/.local/ # Variables for trace generation. PROGRAM="mirror-sync" -VERSION="20260727" +VERSION="20261017" TRACEHOST=$(hostname -f) mirror_hostname=$(hostname -f) DATE_STARTED=$(LC_ALL=POSIX LANG=POSIX date -u -R) @@ -77,6 +77,9 @@ fi # temporary stage logs, ...). The sync functions append to this list as they # create such files so that a Ctrl-C does not leave stale state behind. cleanup_files=() +# Directories to remove if a sync is interrupted, such as the scratch space the +# repository metadata check downloads into. +cleanup_dirs=() # Handle Ctrl-C (SIGINT) and termination (SIGTERM). When run from a terminal # the signal is delivered to the whole process group, so the rsync/aws/git/... @@ -96,6 +99,9 @@ handle_interrupt() { for _f in "${cleanup_files[@]}"; do [[ $_f ]] && rm -f "$_f" done + for _f in "${cleanup_dirs[@]}"; do + [[ $_f ]] && rm -rf "$_f" + done # Remove the pid lock file for this module so a later run is not blocked. [[ $PIDFILE ]] && rm -f "$PIDFILE" @@ -813,8 +819,12 @@ run_post_successful_sync_hook() { fi } -# On failed sync, count failure and exit. +# On failed sync, count failure and exit. An optional reason is printed and +# added to the email, for failures the log alone would not make obvious. post_failed_sync() { + if [[ $1 ]]; then + echo "$1" + fi echo "Sync failed." # Update failure count. @@ -822,7 +832,7 @@ post_failed_sync() { # If failure count is over our maximum defined errors, email about the failure. if ((new_error_count>max_errors)); then - mail_error "Unable to sync with $MODULE, check logs." + mail_error "Unable to sync with $MODULE, check logs.${1:+ $1}" # Remove the error count file so that the count resets. rm -f "$ERRORFILE" @@ -1523,6 +1533,202 @@ read_rsync_config() { eval repo_type="\$${MODULE}_type" } +# Read whether this module's upstream metadata is checked before stage 2. It +# defaults to on for rpm modules synced in two stages, since stage 2 is where +# their metadata is published, and it means nothing without a second stage. +read_repodata_check_config() { + local _default="false" + if [[ $repo_type == "rpm" && $options_stage2 ]]; then + _default="true" + fi + eval repodata_check_enabled="\$${MODULE}_repodata_check" + parse_bool repodata_check_enabled "$_default" "${MODULE}_repodata_check" + if [[ $repodata_check_enabled && ! $options_stage2 ]]; then + echo "${MODULE}_repodata_check needs ${MODULE}_options_stage2, the check runs before stage 2." + exit 1 + fi + eval repodata_check_prune="\$${MODULE}_repodata_check_prune" + repodata_check_prune="${repodata_check_prune:-Packages drpms isos images}" +} + +# Print the newest in a repomd.xml, or 0 when it has none. +repomd_timestamp() { + local _ts + _ts=$(grep -o '[0-9]*' "$1" 2>/dev/null | sed 's///' | sort -n | tail -n 1) + echo "${_ts:-0}" +} + +# Print the locations a repomd.xml names that are relative to its repository. +# Locations under an xml:base or with an absolute path are served from +# elsewhere and are left out. +repomd_locations() { + local _location _href + while IFS= read -r _location; do + if [[ $_location == *xml:base* ]]; then + continue + fi + _href=${_location#*href=\"} + _href=${_href%%\"*} + if [[ $_href == /* || $_href == *://* ]]; then + continue + fi + echo "$_href" + done < <(grep -o ']*>' "$1") +} + +# Check the repository metadata upstream would publish in stage 2, before +# stage 2 runs. Every file a repomd.xml names must exist upstream, and no +# repomd.xml may be older than the copy already published here. An upstream +# caught mid-update, serving a torn tree, or rolled back to an older generation +# fails the check. Stage 1 skips repodata and never deletes, so refusing stage +# 2 keeps the mirror on its last consistent metadata. Returns non-zero, with +# the problems printed, when stage 2 must not run. +# +# A published repomd.xml can differ from upstream's while matching it in size +# and modification time, which rsync takes to mean the file is unchanged, so +# stage 2 would leave it naming files that are gone. When the check passes, +# such a repomd.xml and its signature are given a modification time of 0 so +# stage 2 replaces them. +check_rpm_repodata() { + local _tmp _prune _dir _line _path _repomd _base _href _upstream_ts _local_ts _f + local _checked=0 _torn=0 _older=0 + local -a _stale=() + local -A _upstream_files=() + + _tmp=$(mktemp -d "${tmpDirBase:?}/${MODULE}-repodata.XXXXXX") || { + echo "Could not create a scratch directory under ${tmpDirBase}." + return 1 + } + cleanup_dirs+=("$_tmp") + + # Directories that hold packages rather than metadata are left out of the + # walk to keep the listing fast. + _prune="" + for _dir in ${repodata_check_prune}; do + _prune+=" --exclude '${_dir}/'" + done + + # List every metadata file upstream. The stage 2 options come first, so + # their filters limit the check to what stage 2 would publish. + if ! eval "$sync_timeout" rsync -rl \ + --no-motd \ + --list-only \ + --no-human-readable \ + "${options_stage2:-}" \ + "$_prune" \ + --include "'*/'" \ + --include "'repodata/*'" \ + --exclude "'*'" \ + "'${source:?}'" > "$_tmp/listing"; then + echo "Could not list the upstream repository metadata." + rm -rf "$_tmp" + return 1 + fi + while IFS= read -r _line; do + if [[ $_line =~ ^[-l][^\ ]*\ +[0-9]+\ [0-9/]+\ [0-9:]+\ (.*)$ ]]; then + _path=${BASH_REMATCH[1]} + _upstream_files["${_path%% -> *}"]=1 + fi + done < "$_tmp/listing" + + # Download only the repomd.xml files, by name, so upstream is not walked a + # second time. + for _path in "${!_upstream_files[@]}"; do + if [[ $_path == repodata/repomd.xml || $_path == */repodata/repomd.xml ]]; then + echo "$_path" + fi + done | sort > "$_tmp/repomds" + if [[ ! -s $_tmp/repomds ]]; then + echo "Upstream lists no repodata/repomd.xml, nothing to check." + rm -rf "$_tmp" + return 0 + fi + mkdir "$_tmp/upstream" + if ! $sync_timeout rsync \ + --no-motd \ + --copy-links \ + --files-from="$_tmp/repomds" \ + "${source:?}" "$_tmp/upstream/"; then + echo "Could not download the upstream repomd.xml files." + rm -rf "$_tmp" + return 1 + fi + + while IFS= read -r _repomd; do + _checked=$((_checked+1)) + _base=${_repomd%repodata/repomd.xml} + + # Every relative location must exist upstream. + while IFS= read -r _href; do + if [[ ! ${_upstream_files["${_base}${_href}"]} ]]; then + echo "Torn: ${_repomd} names ${_href}, which upstream does not have." + _torn=$((_torn+1)) + break + fi + done < <(repomd_locations "$_tmp/upstream/$_repomd") + + # The metadata must be no older than what is published here. + if [[ -f ${repo%/}/$_repomd ]]; then + _upstream_ts=$(repomd_timestamp "$_tmp/upstream/$_repomd") + _local_ts=$(repomd_timestamp "${repo%/}/$_repomd") + if (( _upstream_ts > 0 && _local_ts > 0 && _upstream_ts < _local_ts )); then + echo "Older: ${_repomd} is from $(date -u -d "@${_upstream_ts}" '+%F %T') UTC, ours is from $(date -u -d "@${_local_ts}" '+%F %T') UTC." + _older=$((_older+1)) + fi + if ! cmp -s "$_tmp/upstream/$_repomd" "${repo%/}/$_repomd"; then + _stale+=("${repo%/}/$_repomd") + fi + fi + done < "$_tmp/repomds" + rm -rf "$_tmp" + + echo "Checked ${_checked} repomd.xml: ${_torn} torn, ${_older} older than ours, ${#_stale[@]} changed." + if (( _torn > 0 || _older > 0 )); then + return 1 + fi + + # Make sure stage 2 replaces every repomd.xml whose content changed. + for _repomd in "${_stale[@]}"; do + for _f in "$_repomd" "$_repomd".*; do + if [[ -f $_f && ! -L $_f ]]; then + touch -d @0 "$_f" + fi + done + done + return 0 +} + +# Check the repository metadata published here after stage 2: every file a +# repomd.xml names must exist. This catches a torn tree however it came about, +# including one left by an earlier sync, so it is reported instead of being +# recorded as a successful sync. Returns non-zero, with the problems printed, +# when the published tree is torn. +verify_published_repodata() { + local _dir _repomd _base _href _checked=0 _torn=0 + local -a _prune=() + + for _dir in ${repodata_check_prune}; do + _prune+=(-name "$_dir" -o) + done + while IFS= read -r _repomd; do + _checked=$((_checked+1)) + _base=${_repomd%repodata/repomd.xml} + while IFS= read -r _href; do + if [[ ! -e ${_base}${_href} ]]; then + echo "Torn: ${_repomd#"${repo%/}/"} names ${_href}, which is missing here." + _torn=$((_torn+1)) + break + fi + done < <(repomd_locations "$_repomd") + done < <(find "${repo%/}/" \( "${_prune[@]}" -false \) -prune -o -path '*/repodata/repomd.xml' -type f -print) + + echo "Checked ${_checked} published repomd.xml: ${_torn} torn." + if (( _torn > 0 )); then + return 1 + fi + return 0 +} + # Modules based on rsync. rsync_sync() { # Start the module. @@ -1553,6 +1759,7 @@ rsync_sync() { eval pre_stage2_hook="\$${MODULE}_pre_stage2_hook" eval upstream_check="\$${MODULE}_upstream_check" eval time_file_check="\$${MODULE}_time_file_check" + read_repodata_check_config # Check if upstream was updated recently if configured. # This is designed to slow down rsync so we only rsync @@ -1688,6 +1895,16 @@ rsync_sync() { esac done + # Refuse to publish upstream metadata that is torn or older than ours. + if [[ $repodata_check_enabled ]]; then + echo + echo "Checking upstream repository metadata:" + if ! check_rpm_repodata; then + rm -f "$mirror_update_file" + post_failed_sync "Upstream repository metadata failed its check, so stage 2 was skipped and the published metadata was kept." + fi + fi + # Run any hooks. if [[ $pre_stage2_hook ]]; then echo "Executing pre-stage2 hook:" @@ -1726,6 +1943,16 @@ rsync_sync() { rm -f "$mirror_update_file" post_failed_sync fi + + # Confirm the metadata stage 2 left published is whole. + if [[ $repodata_check_enabled ]]; then + echo + echo "Checking published repository metadata:" + if ! verify_published_repodata; then + rm -f "$mirror_update_file" + post_failed_sync "The published repository metadata is torn after stage 2." + fi + fi fi # Count what the sync changed, before the logs it is counted from are