990 lines
30 KiB
Bash
990 lines
30 KiB
Bash
#!/bin/bash
|
|
# This script is designed to handle mirror syncing tasks from external mirrors.
|
|
# Each mirror is handled within a module which can be configured via the configuration file /etc/mirror-sync.conf.
|
|
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/home/mirror/.local/bin:/home/mirror/bin
|
|
|
|
# Variables for trace generation.
|
|
PROGRAM="mirror-sync"
|
|
VERSION="20231107"
|
|
TRACEHOST=$(hostname -f)
|
|
mirror_hostname=$(hostname -f)
|
|
DATE_STARTED=$(LC_ALL=POSIX LANG=POSIX date -u -R)
|
|
INFO_TRIGGER=cron
|
|
if [[ $SUDO_USER ]]; then
|
|
INFO_TRIGGER=ssh
|
|
fi
|
|
|
|
# Pid file temporary path.
|
|
PIDPATH="/tmp"
|
|
PIDSUFFIX="-mirror-sync.pid"
|
|
PIDFILE="" # To be filled by acquire_lock().
|
|
|
|
# Log file.
|
|
LOGPATH="/var/log/mirror-sync"
|
|
LOGFILE="" # To be filled by acquire_lock().
|
|
ERRORFILE="" # To be filled by acquire_lock().
|
|
error_count=0
|
|
max_errors=3
|
|
tmpDirBase="$HOME/tmp"
|
|
# Do not check upstream unless it was updated in the last 5 hours.
|
|
upstream_max_age=18000
|
|
# Update anyway if last check was more than 24 hours ago.
|
|
upstream_timestamp_min=86400
|
|
|
|
# quick-fedora-mirror tool config.
|
|
QFM_GIT="https://pagure.io/quick-fedora-mirror.git"
|
|
QFM_PATH="$HOME/quick-fedora-mirror"
|
|
QFM_BIN="$QFM_PATH/quick-fedora-mirror"
|
|
|
|
# For installing Jigdo
|
|
JIGDO_SOURCE_URL="http://deb.debian.org/debian/pool/main/j/jigdo/jigdo_0.8.0.orig.tar.xz"
|
|
JIGDO_FILE_BIN="$HOME/bin/jigdo-file"
|
|
JIGDO_MIRROR_BIN="$HOME/bin/jigdo-mirror"
|
|
jigdoConf="$HOME/etc/jigdo/jigdo-mirror.conf"
|
|
|
|
# Prevent run as root.
|
|
if (( EUID == 0 )); then
|
|
echo "Do not mirror as root."
|
|
exit 1
|
|
fi
|
|
|
|
# Load the required configuration file or quit.
|
|
if [[ -f /etc/mirror-sync.conf ]]; then
|
|
# shellcheck disable=SC1090
|
|
source /etc/mirror-sync.conf
|
|
else
|
|
echo "No configuration file defined, please setup a proper configuration file."
|
|
exit 1
|
|
fi
|
|
|
|
# Print the help for this command.
|
|
print_help() {
|
|
echo "Mirror Sync"
|
|
echo
|
|
echo "Usage:"
|
|
echo "$0 [--help|--update-support-utilities] {module} [--force]"
|
|
echo
|
|
echo "Available modules:"
|
|
for MODULE in ${MODULES:?}; do
|
|
echo "$MODULE"
|
|
done
|
|
exit
|
|
}
|
|
|
|
# Send email to admins about error.
|
|
mail_error() {
|
|
if [[ -z $MAILTO ]]; then
|
|
echo "MAILTO is undefined."
|
|
return
|
|
fi
|
|
{
|
|
cat <<EOF
|
|
Subject: ${PROGRAM} Error
|
|
To: ${MAILTO}
|
|
Auto-Submitted: auto-generated
|
|
MIME-Version: 1.0
|
|
Content-Type: text/plain
|
|
|
|
Host: $(hostname -f)
|
|
Module: ${MODULE}
|
|
Logfile: ${LOGFILE}
|
|
|
|
$*
|
|
EOF
|
|
} | sendmail -i -t
|
|
}
|
|
|
|
# Installs quick-fedora-mirror and updates.
|
|
quick_fedora_mirror_install() {
|
|
if ! [[ -f $QFM_BIN ]]; then
|
|
echo "quick-fedora-mirror is not on this system, attempting to get it"
|
|
[[ -e $QFM_PATH ]] && rm -Rf "$QFM_PATH"
|
|
git clone "$QFM_GIT" "$QFM_PATH"
|
|
if ! [[ -f $QFM_BIN ]]; then
|
|
echo "Failed to get quick-fedora-mirror!"
|
|
exit 1
|
|
fi
|
|
fi
|
|
(
|
|
if [[ $1 == "-u" ]]; then
|
|
if ! cd "$QFM_PATH"; then
|
|
echo "Unable to enter QFM path."
|
|
exit 1
|
|
fi
|
|
if ! git pull; then
|
|
echo "Unable to update QFM."
|
|
exit 1
|
|
fi
|
|
fi
|
|
)
|
|
}
|
|
|
|
# Installs jigdo image tool.
|
|
jigdo_install() {
|
|
if [[ $1 == "-u" ]] || ! [[ -e $JIGDO_FILE_BIN ]]; then
|
|
if ! cd "$HOME"; then
|
|
echo "Unable to access home dir."
|
|
exit 1
|
|
fi
|
|
if [[ ! -d bin ]]; then
|
|
mkdir -p bin
|
|
fi
|
|
if ! wget "$JIGDO_SOURCE_URL" -O jigdo.tar.xz; then
|
|
echo "Unable to download jigdo utility."
|
|
exit 1
|
|
fi
|
|
if ! tar -xvf jigdo.tar.xz; then
|
|
echo "Unable to unarchive jigdo."
|
|
exit 1
|
|
fi
|
|
rm -f jigdo.tar.xz
|
|
(
|
|
if ! cd jigdo-*/; then
|
|
echo "Unable to enter extracted archive."
|
|
exit 1
|
|
fi
|
|
cat > jigdo.patch <<'EOF'
|
|
--- src/util/sha256sum.hh 2019-11-19 10:43:22.000000000 -0500
|
|
+++ src-fix/util/sha256sum.hh 2023-04-19 16:33:40.840831304 -0400
|
|
@@ -27,6 +27,7 @@
|
|
#include <cstring>
|
|
#include <iosfwd>
|
|
#include <string>
|
|
+#include <stdint.h>
|
|
|
|
#include <bstream.hh>
|
|
#include <debug.hh>
|
|
EOF
|
|
patch -u src/util/sha256sum.hh -i jigdo.patch
|
|
if ! ./configure --prefix="$HOME"; then
|
|
echo "Unable to configure jigdo."
|
|
exit 1
|
|
fi
|
|
|
|
# Build fails first few times due to docs, but clears after a few builds.
|
|
if ! make; then
|
|
if ! make; then
|
|
make
|
|
fi
|
|
fi
|
|
make install
|
|
)
|
|
fi
|
|
}
|
|
|
|
# Updates the mirror support utilties on server with upstream.
|
|
update_support_utilities() {
|
|
quick_fedora_mirror_install -u
|
|
jigdo_install -u
|
|
}
|
|
|
|
# Acquire a sync lock for this command.
|
|
acquire_lock() {
|
|
MODULE=$1
|
|
# Pid file for this module sync.
|
|
PIDFILE="${PIDPATH}/${MODULE}${PIDSUFFIX}"
|
|
LOGFILE="${LOGPATH}/${MODULE}.log"
|
|
ERRORFILE="${LOGPATH}/${MODULE}.error_count"
|
|
if [[ -e $ERRORFILE ]]; then
|
|
error_count=$(cat "$ERRORFILE")
|
|
fi
|
|
|
|
# Redirect stdout to both stdout and log file.
|
|
exec 1> >(tee -a "$LOGFILE")
|
|
# Redirect errors to stdout so they also are logged.
|
|
exec 2>&1
|
|
|
|
# Check existing pid file.
|
|
if [[ -f $PIDFILE ]]; then
|
|
PID=$(cat "$PIDFILE")
|
|
# Prevent double locks.
|
|
if [[ $PID == "$BASHPID" ]]; then
|
|
echo "Double lock detected."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if PID is active.
|
|
if ps -p "$PID" >/dev/null; then
|
|
echo "A sync is already in progress for ${MODULE} with pid ${PID}."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Create a new pid file for this process.
|
|
echo $BASHPID >"$PIDFILE"
|
|
|
|
# On exit, remove pid file.
|
|
trap 'rm -f "$PIDFILE"' EXIT
|
|
}
|
|
|
|
log_start_header() {
|
|
echo
|
|
echo "=========================================="
|
|
echo "Starting execution: $(date +"%Y-%m-%d %T")"
|
|
echo "=========================================="
|
|
echo
|
|
}
|
|
|
|
log_end_header() {
|
|
echo
|
|
echo "=========================================="
|
|
echo "Execution complete: $(date +"%Y-%m-%d %T")"
|
|
echo "=========================================="
|
|
}
|
|
|
|
# Sync git based mirrors.
|
|
git_sync() {
|
|
MODULE=$1
|
|
acquire_lock "$MODULE"
|
|
|
|
# Read the configuration for this module.
|
|
eval repo="\$${MODULE}_repo"
|
|
eval timestamp="\$${MODULE}_timestamp"
|
|
eval options="\$${MODULE}_options"
|
|
|
|
# If configuration is not set, exit.
|
|
if [[ ! $repo ]]; then
|
|
echo "No configuration exists for ${MODULE}"
|
|
exit 1
|
|
fi
|
|
log_start_header
|
|
|
|
(
|
|
# Do a git pull within the repo folder to sync.
|
|
if ! cd "${repo:?}"; then
|
|
echo "Failed to access '${repo:?}' git repository."
|
|
exit 1
|
|
fi
|
|
eval git pull "$options"
|
|
RT=${PIPESTATUS[0]}
|
|
if (( RT == 0 )); then
|
|
date +%s > "${timestamp:?}"
|
|
if [[ -e $ERRORFILE ]]; then
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
else
|
|
error_count=$((error_count+1))
|
|
if ((error_count>max_errors)); then
|
|
mail_error "Unable to sync with git, check logs."
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
echo "$error_count" > "$ERRORFILE"
|
|
fi
|
|
)
|
|
|
|
log_end_header
|
|
}
|
|
|
|
# Sync AWS S3 bucket based mirrors.
|
|
aws_sync() {
|
|
MODULE=$1
|
|
acquire_lock "$MODULE"
|
|
|
|
# Read the configuration for this module.
|
|
eval repo="\$${MODULE}_repo"
|
|
eval timestamp="\$${MODULE}_timestamp"
|
|
eval bucket="\$${MODULE}_aws_bucket"
|
|
eval AWS_ACCESS_KEY_ID="\$${MODULE}_aws_access_key"
|
|
eval AWS_SECRET_ACCESS_KEY="\$${MODULE}_aws_secret_key"
|
|
eval AWS_ENDPOINT_URL="\$${MODULE}_aws_endpoint_url"
|
|
eval options="\$${MODULE}_options"
|
|
export AWS_ACCESS_KEY_ID
|
|
export AWS_SECRET_ACCESS_KEY
|
|
|
|
# If configuration is not set, exit.
|
|
if [[ ! $repo ]]; then
|
|
echo "No configuration exists for ${MODULE}"
|
|
exit 1
|
|
fi
|
|
log_start_header
|
|
|
|
if [[ -n $AWS_ENDPOINT_URL ]]; then
|
|
options="$options --endpoint-url='$AWS_ENDPOINT_URL'"
|
|
fi
|
|
|
|
# Run AWS client to sync the S3 bucket.
|
|
eval timeout 1d aws s3 sync \
|
|
--no-follow-symlinks \
|
|
--delete \
|
|
"$options" \
|
|
"'${bucket:?}'" "'${repo:?}'"
|
|
RT=${PIPESTATUS[0]}
|
|
if (( RT == 0 )); then
|
|
date +%s > "${timestamp:?}"
|
|
if [[ -e $ERRORFILE ]]; then
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
else
|
|
error_count=$((error_count+1))
|
|
if ((error_count>max_errors)); then
|
|
mail_error "Unable to sync with aws, check logs."
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
echo "$error_count" > "$ERRORFILE"
|
|
fi
|
|
|
|
log_end_header
|
|
}
|
|
|
|
# Sync using FTP.
|
|
ftp_sync() {
|
|
MODULE=$1
|
|
acquire_lock "$MODULE"
|
|
|
|
# Read the configuration for this module.
|
|
eval repo="\$${MODULE}_repo"
|
|
eval timestamp="\$${MODULE}_timestamp"
|
|
eval source="\$${MODULE}_source"
|
|
eval options="\$${MODULE}_options"
|
|
|
|
# If configuration is not set, exit.
|
|
if [[ ! $repo ]]; then
|
|
echo "No configuration exists for ${MODULE}"
|
|
exit 1
|
|
fi
|
|
log_start_header
|
|
|
|
# Run AWS client to sync the S3 bucket.
|
|
timeout 1d lftp <<< "mirror -v --delete --no-perms $options '${source:?}' '${repo:?}'"
|
|
RT=${PIPESTATUS[0]}
|
|
if (( RT == 0 )); then
|
|
date +%s > "${timestamp:?}"
|
|
if [[ -e $ERRORFILE ]]; then
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
else
|
|
error_count=$((error_count+1))
|
|
if ((error_count>max_errors)); then
|
|
mail_error "Unable to sync with lftp, check logs."
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
echo "$error_count" > "$ERRORFILE"
|
|
fi
|
|
|
|
log_end_header
|
|
}
|
|
|
|
# Sync using wget.
|
|
wget_sync() {
|
|
MODULE=$1
|
|
acquire_lock "$MODULE"
|
|
|
|
# Read the configuration for this module.
|
|
eval repo="\$${MODULE}_repo"
|
|
eval timestamp="\$${MODULE}_timestamp"
|
|
eval source="\$${MODULE}_source"
|
|
eval options="\$${MODULE}_options"
|
|
|
|
if [[ -z $options ]]; then
|
|
options="--mirror --no-host-directories --no-parent"
|
|
fi
|
|
|
|
# If configuration is not set, exit.
|
|
if [[ ! $repo ]]; then
|
|
echo "No configuration exists for ${MODULE}"
|
|
exit 1
|
|
fi
|
|
log_start_header
|
|
|
|
(
|
|
# Make sure the repo directory exists and we are in it.
|
|
if ! [[ -e $repo ]]; then
|
|
mkdir -p "$repo"
|
|
fi
|
|
|
|
if ! cd "$repo"; then
|
|
echo "Unable to enter repo directory."
|
|
fi
|
|
|
|
# Run wget with configured options.
|
|
eval timeout 1d wget "$options" "'${source:?}'"
|
|
RT=${PIPESTATUS[0]}
|
|
if (( RT == 0 )); then
|
|
date +%s > "${timestamp:?}"
|
|
if [[ -e $ERRORFILE ]]; then
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
else
|
|
error_count=$((error_count+1))
|
|
if ((error_count>max_errors)); then
|
|
mail_error "Unable to sync with lftp, check logs."
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
echo "$error_count" > "$ERRORFILE"
|
|
fi
|
|
)
|
|
|
|
log_end_header
|
|
}
|
|
|
|
# Jigdo hook - builds iso images from jigdo files.
|
|
jigdo_hook() {
|
|
jigdo_install
|
|
currentVersion=$(ls -l "${repo}/current")
|
|
currentVersion="${currentVersion##* -> }"
|
|
versionDir="$(realpath "$repo")/${currentVersion}"
|
|
for a in "$versionDir"/*/; do
|
|
arch=$(basename "$a")
|
|
sets=$(cat "${repo}/project/build/${currentVersion}/${arch}")
|
|
for s in $sets; do
|
|
jigdoDir="${repo}/${currentVersion}/${arch}/jigdo-${s}"
|
|
imageDir="${repo}/${currentVersion}/${arch}/iso-${s}"
|
|
if [[ ! -d $imageDir ]]; then
|
|
mkdir -p "$imageDir"
|
|
fi
|
|
# Sums are now SHA256SUMS and SHA512SUMS.
|
|
cp -a "${jigdoDir}"/*SUMS* "${imageDir}/"
|
|
cat >"${jigdoConf:?}.${arch}.${s}" <<EOF
|
|
LOGROTATE=14
|
|
jigdoFile="$JIGDO_FILE_BIN --cache=\$tmpDir/jigdo-cache.db --cache-expiry=1w --report=noprogress --no-check-files"
|
|
debianMirror="file:${repo}"
|
|
nonusMirror="file:/tmp"
|
|
include='.' # include all files,
|
|
exclude='^$' # then exclude none
|
|
jigdoDir=${jigdoDir}"
|
|
imageDir=${imageDir}
|
|
tmpDir=${tmpDirBase:?}/${arch}.${s}
|
|
#logfile=${LOGPATH}/${MODULE}-${arch}.${s}.log
|
|
EOF
|
|
echo "Running jigdo for ${arch}.${s}"
|
|
$JIGDO_MIRROR_BIN "${jigdoConf:?}.${arch}.${s}"
|
|
done
|
|
done
|
|
}
|
|
|
|
# Pull a field from a trace file or rsync stats.
|
|
extract_trace_field() {
|
|
value=$(awk -F': ' "\$1==\"$1\" {print \$2; exit}" "$2" 2>/dev/null)
|
|
[[ $value ]] || return 1
|
|
echo "$value"
|
|
}
|
|
|
|
# Build trace content.
|
|
build_trace_content() {
|
|
LC_ALL=POSIX LANG=POSIX date -u
|
|
rfc822date=$(LC_ALL=POSIX LANG=POSIX date -u -R)
|
|
echo "Date: ${rfc822date}"
|
|
echo "Date-Started: ${DATE_STARTED}"
|
|
|
|
if [[ -e $TRACEFILE_MASTER ]]; then
|
|
echo "Archive serial: $(extract_trace_field 'Archive serial' "$TRACE_MASTER_FILE" || echo unknown )"
|
|
fi
|
|
|
|
echo "Used ${PROGRAM} version: ${VERSION}"
|
|
echo "Creator: ${PROGRAM} ${VERSION}"
|
|
echo "Running on host: ${TRACEHOST}"
|
|
|
|
if [[ ${INFO_MAINTAINER:-} ]]; then
|
|
echo "Maintainer: ${INFO_MAINTAINER}"
|
|
fi
|
|
if [[ ${INFO_SPONSOR:-} ]]; then
|
|
echo "Sponsor: ${INFO_SPONSOR}"
|
|
fi
|
|
if [[ ${INFO_COUNTRY:-} ]]; then
|
|
echo "Country: ${INFO_COUNTRY}"
|
|
fi
|
|
if [[ ${INFO_LOCATION:-} ]]; then
|
|
echo "Location: ${INFO_LOCATION}"
|
|
fi
|
|
if [[ ${INFO_THROUGHPUT:-} ]]; then
|
|
echo "Throughput: ${INFO_THROUGHPUT}"
|
|
fi
|
|
if [[ ${INFO_TRIGGER:-} ]]; then
|
|
echo "Trigger: ${INFO_TRIGGER}"
|
|
fi
|
|
|
|
# Depending on repo type, find archetectures supported.
|
|
ARCH_REGEX='(source|SRPMS|amd64|mips64el|mipsel|i386|x86_64|aarch64|ppc64le|ppc64el|s390x|armhf)'
|
|
if [[ $repo_type == "deb" ]]; then
|
|
ARCH=$(find "${repo}/dists" \( -name 'Packages.*' -o -name 'Sources.*' \) 2>/dev/null |
|
|
sed -Ene 's#.*/binary-([^/]+)/Packages.*#\1#p; s#.*/(source)/Sources.*#\1#p' |
|
|
sort -u | tr '\n' ' ')
|
|
if [[ $ARCH ]]; then
|
|
echo "Architectures: ${ARCH}"
|
|
fi
|
|
elif [[ $repo_type == "rpm" ]]; then
|
|
ARCH=$(find "$repo" -name 'repomd.xml' 2>/dev/null |
|
|
grep -Po "$ARCH_REGEX" |
|
|
sort -u | tr '\n' ' ')
|
|
if [[ $ARCH ]]; then
|
|
echo "Architectures: ${ARCH}"
|
|
fi
|
|
elif [[ $repo_type == "iso" ]]; then
|
|
ARCH=$(find "$repo" -name '*.iso' 2>/dev/null |
|
|
grep -Po "$ARCH_REGEX" |
|
|
sort -u | tr '\n' ' ')
|
|
if [[ $ARCH ]]; then
|
|
echo "Architectures: ${ARCH}"
|
|
fi
|
|
elif [[ $repo_type == "source" ]]; then
|
|
echo "Architectures: source"
|
|
fi
|
|
echo "Architectures-Configuration: ${arch_configurations:-ALL}"
|
|
|
|
echo "Upstream-mirror: ${RSYNC_HOST:-unknown}"
|
|
|
|
# Total bytes synced per rsync stage.
|
|
total=0
|
|
if [[ -f $LOGFILE_SYNC ]]; then
|
|
for bytes in $(sed -Ene 's/(^|.* )sent ([0-9]+) bytes received ([0-9]+) bytes.*/\3/p' "$LOGFILE_SYNC"); do
|
|
total=$(( total + bytes ))
|
|
done
|
|
elif [[ -f $LOGFILE_STAGE1 ]]; then
|
|
bytes=$(sed -Ene 's/(^|.* )sent ([0-9]+) bytes received ([0-9]+) bytes.*/\3/p' "$LOGFILE_STAGE1")
|
|
total=$(( total + bytes ))
|
|
fi
|
|
if [[ -f $LOGFILE_STAGE2 ]]; then
|
|
bytes=$(sed -Ene 's/(^|.* )sent ([0-9]+) bytes received ([0-9]+) bytes.*/\3/p' "$LOGFILE_STAGE2")
|
|
total=$(( total + bytes ))
|
|
fi
|
|
if (( total > 0 )); then
|
|
echo "Total bytes received in rsync: ${total}"
|
|
fi
|
|
|
|
# Calculate time per rsync stage and print both stages if both were started.
|
|
if [[ $sync_started ]]; then
|
|
STATS_TOTAL_RSYNC_TIME1=$(( sync_ended - sync_started ))
|
|
total_time=$STATS_TOTAL_RSYNC_TIME1
|
|
elif [[ $stage1_started ]]; then
|
|
STATS_TOTAL_RSYNC_TIME1=$(( stage1_ended - stage1_started ))
|
|
total_time=$STATS_TOTAL_RSYNC_TIME1
|
|
fi
|
|
if [[ $stage2_started ]]; then
|
|
STATS_TOTAL_RSYNC_TIME2=$(( stage2_ended - stage2_started ))
|
|
total_time=$(( total_time + STATS_TOTAL_RSYNC_TIME2 ))
|
|
echo "Total time spent in stage1 rsync: ${STATS_TOTAL_RSYNC_TIME1}"
|
|
echo "Total time spent in stage2 rsync: ${STATS_TOTAL_RSYNC_TIME2}"
|
|
fi
|
|
echo "Total time spent in rsync: ${total_time}"
|
|
if (( total_time != 0 )); then
|
|
rate=$(( total / total_time ))
|
|
echo "Average rate: ${rate} B/s"
|
|
fi
|
|
}
|
|
|
|
# Save trace file.
|
|
save_trace_file() {
|
|
# Trace file/dir paths.
|
|
TRACE_DIR="${repo}/project/trace"
|
|
mkdir -p "$TRACE_DIR"
|
|
TRACE_FILE="${TRACE_DIR}/${mirror_hostname:?}"
|
|
TRACE_MASTER_FILE="${TRACE_DIR}/master"
|
|
TRACE_HIERARCHY="${TRACE_DIR}/_hierarchy"
|
|
|
|
# Parse the rsync host from the source.
|
|
RSYNC_HOST=${source/rsync:\/\//}
|
|
RSYNC_HOST=${RSYNC_HOST%%:*}
|
|
RSYNC_HOST=${RSYNC_HOST%%/*}
|
|
|
|
# Build trace and save to file.
|
|
build_trace_content > "${TRACE_FILE}.new"
|
|
mv "${TRACE_FILE}.new" "$TRACE_FILE"
|
|
|
|
# Build heirarchy file.
|
|
{
|
|
if [[ -e "${TRACE_HIERARCHY}.mirror" ]]; then
|
|
cat "${TRACE_HIERARCHY}.mirror"
|
|
fi
|
|
echo "$(basename "$TRACE_FILE") $mirror_hostname $TRACEHOST ${RSYNC_HOST:-unknown}"
|
|
} > "${TRACE_HIERARCHY}.new"
|
|
mv "${TRACE_HIERARCHY}.new" "$TRACE_HIERARCHY"
|
|
cp "$TRACE_HIERARCHY" "${TRACE_HIERARCHY}.mirror"
|
|
|
|
# Output all traces to _traces file. Disabling shell check because the glob in this case is used right.
|
|
# shellcheck disable=SC2035
|
|
(cd "$TRACE_DIR" && find * -type f \! -name "_*") > "$TRACE_DIR/_traces"
|
|
}
|
|
|
|
# Modules based on rsync.
|
|
rsync_sync() {
|
|
MODULE=$1
|
|
shift
|
|
|
|
# Check for any arguments.
|
|
force=0
|
|
while (( $# > 0 )); do
|
|
case $1 in
|
|
# Force rsync, ignore upstream check.
|
|
-f|--force)
|
|
force=1
|
|
shift
|
|
;;
|
|
*)
|
|
echo "Unknown option $1"
|
|
echo
|
|
print_help "$@"
|
|
;;
|
|
esac
|
|
done
|
|
acquire_lock "$MODULE"
|
|
|
|
# Read the configuration for this module.
|
|
eval repo="\$${MODULE}_repo"
|
|
eval pre_hook="\$${MODULE}_pre_hook"
|
|
eval timestamp="\$${MODULE}_timestamp"
|
|
eval source="\$${MODULE}_source"
|
|
eval options="\$${MODULE}_options"
|
|
eval options_stage2="\$${MODULE}_options_stage2"
|
|
eval pre_stage2_hook="\$${MODULE}_pre_stage2_hook"
|
|
eval upstream_check="\$${MODULE}_upstream_check"
|
|
eval report_mirror="\$${MODULE}_report_mirror"
|
|
eval RSYNC_PASSWORD="\$${MODULE}_rsync_password"
|
|
if [[ $RSYNC_PASSWORD ]]; then
|
|
export RSYNC_PASSWORD
|
|
fi
|
|
eval post_hook="\$${MODULE}_post_hook"
|
|
eval arch_configurations="\$${MODULE}_arch_configurations"
|
|
eval repo_type="\$${MODULE}_type"
|
|
|
|
# If configuration is not set, exit.
|
|
if [[ ! $repo ]]; then
|
|
echo "No configuration exists for ${MODULE}"
|
|
exit 1
|
|
fi
|
|
log_start_header
|
|
|
|
# Check if upstream was updated recently if configured.
|
|
# This is designed to slow down rsync so we only rsync
|
|
# when we detect its needed or when last rsync was a long time ago.
|
|
if [[ $upstream_check ]] && (( force == 0 )); then
|
|
now=$(date +%s)
|
|
last_timestamp=$(cat "${timestamp:?}")
|
|
|
|
# If last update was not that long ago, we should check if upstream was updated recently.
|
|
if [[ $((now-last_timestamp)) -lt ${upstream_timestamp_min:?} ]]; then
|
|
echo "Checking upstream's last modified."
|
|
|
|
# Get the last modified date.
|
|
IFS=': ' read -r _ last_modified < <(curl -sI HEAD "${upstream_check:?}" | grep Last-Modified)
|
|
last_modified_unix=$(date -u +%s -d "$last_modified")
|
|
|
|
# If last modified is greater than our max age, it wasn't modified recently and we should not rsync.
|
|
if (( now-last_modified_unix > ${upstream_max_age:-0} )); then
|
|
echo "Skipping sync as upstream wasn't updated recently."
|
|
exit 88
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Run any hooks.
|
|
if [[ $pre_hook ]]; then
|
|
echo "Executing pre-hook:"
|
|
eval "$pre_hook"
|
|
fi
|
|
|
|
# Add arguments from configurations.
|
|
extra_args="${options:-}"
|
|
# If 2 stage, we do not want to delete in stage 1.
|
|
if [[ ! $options_stage2 ]]; then
|
|
extra_args+=" --delete --delete-after"
|
|
echo "Running rsync:"
|
|
else
|
|
echo "Running rsync stage 1:"
|
|
fi
|
|
|
|
# Create archive update file.
|
|
mirror_update_file="${repo:?}/Archive-Update-in-Progress-${mirror_hostname:?}"
|
|
touch "$mirror_update_file"
|
|
LOGFILE_STAGE1="${LOGFILE}.stage1"
|
|
echo -n > "$LOGFILE_STAGE1"
|
|
LOGFILE_STAGE2="${LOGFILE}.stage2"
|
|
echo -n > "$LOGFILE_STAGE2"
|
|
|
|
# Run the rsync. Using eval here so extra_args expands and is used as arguments.
|
|
stage1_started=$(date +%s)
|
|
eval timeout 1d rsync -avH \
|
|
--human-readable \
|
|
--progress \
|
|
--safe-links \
|
|
--delay-updates \
|
|
--stats \
|
|
--no-human-readable \
|
|
--itemize-changes \
|
|
--timeout=10800 \
|
|
"$extra_args" \
|
|
--exclude "Archive-Update-in-Progress-${mirror_hostname:?}" \
|
|
--exclude "project/trace/${mirror_hostname:?}" \
|
|
"'${source:?}'" "'${repo:?}'" | tee -a "$LOGFILE_STAGE1"
|
|
RT=${PIPESTATUS[0]}
|
|
stage1_ended=$(date +%s)
|
|
|
|
# Check if run was successful.
|
|
if [[ $(grep -c '^total size is' "$LOGFILE_STAGE1") -ne 1 ]]; then
|
|
echo "Rsync failed."
|
|
error_count=$((error_count+1))
|
|
if ((error_count>max_errors)); then
|
|
mail_error "Unable to sync with rsync, check logs."
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
echo "$error_count" > "$ERRORFILE"
|
|
exit 1
|
|
fi
|
|
|
|
# If 2 stage, perform second stage.
|
|
if [[ $options_stage2 ]]; then
|
|
# Check if upstream is currently updating.
|
|
for aupfile in "${repo:?}/Archive-Update-in-Progress-"*; do
|
|
case "$aupfile" in
|
|
"$mirror_update_file")
|
|
:
|
|
;;
|
|
*)
|
|
if [[ -f $aupfile ]]; then
|
|
# Remove the file, it will be synced again if
|
|
# upstream is still not done
|
|
rm -f "$aupfile"
|
|
else
|
|
echo "AUIP file '${aupfile}' is not really a file, weird"
|
|
fi
|
|
echo "Upstream is currently updating their repo, skipping second stage for now."
|
|
rm -f "$mirror_update_file"
|
|
exit 0
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Run any hooks.
|
|
if [[ $pre_stage2_hook ]]; then
|
|
echo "Executing pre-stage2 hook:"
|
|
eval "$pre_stage2_hook"
|
|
fi
|
|
|
|
# Add stage 2 options from configurations.
|
|
extra_args="${options_stage2:-}"
|
|
|
|
echo
|
|
echo "Running rsync stage 2:"
|
|
|
|
# Run the rsync. Using eval here so extra_args expands and is used as arguments.
|
|
stage2_started=$(date +%s)
|
|
eval timeout 1d rsync -avH \
|
|
--human-readable \
|
|
--progress \
|
|
--safe-links \
|
|
--delete \
|
|
--delete-after \
|
|
--delay-updates \
|
|
--stats \
|
|
--no-human-readable \
|
|
--itemize-changes \
|
|
--timeout=10800 \
|
|
"$extra_args" \
|
|
--exclude "Archive-Update-in-Progress-${mirror_hostname:?}" \
|
|
--exclude "project/trace/${mirror_hostname:?}" \
|
|
"'${source:?}'" "'${repo:?}'" | tee -a "$LOGFILE_STAGE2"
|
|
RT=${PIPESTATUS[0]}
|
|
stage2_ended=$(date +%s)
|
|
|
|
# Check if run was successful.
|
|
if [[ $(grep -c '^total size is' "$LOGFILE_STAGE2") -ne 1 ]]; then
|
|
echo "Rsync stage 2 failed."
|
|
error_count=$((error_count+1))
|
|
if ((error_count>max_errors)); then
|
|
mail_error "Unable to sync with rsync stage 2, check logs."
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
echo "$error_count" > "$ERRORFILE"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# At this point we are successful, update timestamp of last sync.
|
|
date +%s > "${timestamp:?}"
|
|
if [[ -e $ERRORFILE ]]; then
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
|
|
# Run any hooks.
|
|
if [[ $post_hook ]]; then
|
|
echo "Executing post hook:"
|
|
eval "$post_hook"
|
|
fi
|
|
|
|
# Save trace information.
|
|
if [[ $repo_type ]]; then
|
|
save_trace_file
|
|
fi
|
|
rm -f "$LOGFILE_STAGE1"
|
|
rm -f "$LOGFILE_STAGE2"
|
|
|
|
# Remove archive update file.
|
|
rm -f "$mirror_update_file"
|
|
|
|
# If report mirror configuration file provided, run report mirror.
|
|
if [[ $report_mirror ]]; then
|
|
echo
|
|
echo "Reporting mirror update:"
|
|
/bin/report_mirror -c "${report_mirror:?}"
|
|
fi
|
|
|
|
log_end_header
|
|
}
|
|
|
|
# Modules based on quick-fedora-mirror.
|
|
quick_fedora_mirror_sync() {
|
|
MODULE=$1
|
|
acquire_lock "$MODULE"
|
|
|
|
# Read the configuration for this module.
|
|
eval repo="\$${MODULE}_repo"
|
|
eval pre_hook="\$${MODULE}_pre_hook"
|
|
eval timestamp="\$${MODULE}_timestamp"
|
|
eval source="\$${MODULE}_source"
|
|
eval master_module="\$${MODULE}_master_module"
|
|
eval module_mapping="\$${MODULE}_module_mapping"
|
|
eval mirror_manager_mapping="\$${MODULE}_mirror_manager_mapping"
|
|
eval modules="\$${MODULE}_modules"
|
|
eval options="\$${MODULE}_options"
|
|
eval filterexp="\$${MODULE}_filterexp"
|
|
eval rsync_options="\$${MODULE}_rsync_options"
|
|
eval report_mirror="\$${MODULE}_report_mirror"
|
|
eval RSYNC_PASSWORD="\$${MODULE}_rsync_password"
|
|
if [[ $RSYNC_PASSWORD ]]; then
|
|
export RSYNC_PASSWORD
|
|
fi
|
|
eval post_hook="\$${MODULE}_post_hook"
|
|
eval arch_configurations="\$${MODULE}_arch_configurations"
|
|
eval repo_type="\$${MODULE}_type"
|
|
|
|
# If configuration is not set, exit.
|
|
if [[ ! $repo ]]; then
|
|
echo "No configuration exists for ${MODULE}"
|
|
exit 1
|
|
fi
|
|
log_start_header
|
|
|
|
quick_fedora_mirror_install
|
|
|
|
conf_path="${QFM_PATH}/${MODULE}_qfm.conf"
|
|
cat <<EOF > "$conf_path"
|
|
DESTD="$repo"
|
|
TIMEFILE="${LOGPATH}/${MODULE}_timefile.txt"
|
|
REMOTE="$source"
|
|
MODULES=(${modules:?})
|
|
FILTEREXP='${filterexp:-}'
|
|
VERBOSE=7
|
|
LOGITEMS=aeEl
|
|
RSYNCOPTS=(-aSH -f 'R .~tmp~' --stats --no-human-readable --preallocate --delay-updates ${rsync_options:-} --out-format='@ %i %n%L')
|
|
EOF
|
|
if [[ $master_module ]]; then
|
|
echo "MASTERMODULE='$master_module'" >> "$conf_path"
|
|
fi
|
|
if [[ $module_mapping ]]; then
|
|
echo "MODULEMAPPING=($module_mapping)" >> "$conf_path"
|
|
fi
|
|
if [[ $mirror_manager_mapping ]]; then
|
|
echo "MIRRORMANAGERMAPPING=($mirror_manager_mapping)" >> "$conf_path"
|
|
fi
|
|
|
|
# Run any hooks.
|
|
if [[ $pre_hook ]]; then
|
|
echo "Executing pre-hook:"
|
|
eval "$pre_hook"
|
|
fi
|
|
|
|
# Create archive update file.
|
|
mirror_update_file="${repo:?}/Archive-Update-in-Progress-${mirror_hostname:?}"
|
|
touch "$mirror_update_file"
|
|
LOGFILE_SYNC="${LOGFILE}.sync"
|
|
echo -n > "$LOGFILE_SYNC"
|
|
|
|
# Add arguments from configurations.
|
|
extra_args="${options:-}"
|
|
|
|
# Run the rsync. Using eval here so extra_args expands and is used as arguments.
|
|
sync_started=$(date +%s)
|
|
eval timeout 1d "$QFM_BIN" \
|
|
-c "'$conf_path'" \
|
|
"$extra_args" | tee -a "$LOGFILE_SYNC"
|
|
RT=${PIPESTATUS[0]}
|
|
sync_ended=$(date +%s)
|
|
|
|
# Check if run was successful.
|
|
if [[ $(grep -c '^total size is' "$LOGFILE_SYNC") -lt 1 ]]; then
|
|
echo "Rsync failed."
|
|
error_count=$((error_count+1))
|
|
if ((error_count>max_errors)); then
|
|
mail_error "Unable to sync with rsync, check logs."
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
echo "$error_count" > "$ERRORFILE"
|
|
exit 1
|
|
fi
|
|
|
|
# At this point we are successful, update timestamp of last sync.
|
|
date +%s > "${timestamp:?}"
|
|
if [[ -e $ERRORFILE ]]; then
|
|
rm -f "$ERRORFILE"
|
|
fi
|
|
|
|
# Run any hooks.
|
|
if [[ $post_hook ]]; then
|
|
echo "Executing post hook:"
|
|
eval "$post_hook"
|
|
fi
|
|
|
|
# Save trace information.
|
|
if [[ $repo_type ]]; then
|
|
save_trace_file
|
|
fi
|
|
rm -f "$LOGFILE_SYNC"
|
|
|
|
# Remove archive update file.
|
|
rm -f "$mirror_update_file"
|
|
|
|
# If report mirror configuration file provided, run report mirror.
|
|
if [[ $report_mirror ]]; then
|
|
echo
|
|
echo "Reporting mirror update:"
|
|
/bin/report_mirror -c "${report_mirror:?}"
|
|
fi
|
|
|
|
log_end_header
|
|
}
|
|
|
|
# If no arugments are provided, we can print help.
|
|
if (( $# < 1 )); then
|
|
print_help "$@"
|
|
fi
|
|
|
|
# Parse arguments.
|
|
while (( $# > 0 )); do
|
|
case "$1" in
|
|
# Installs utilities used by this script which are not available in the standard repositories.
|
|
-u|--update-support-utilities)
|
|
update_support_utilities
|
|
exit 0
|
|
;;
|
|
# If help is requested, print it.
|
|
-h|h|help|--help)
|
|
print_help "$@"
|
|
;;
|
|
# Default to rsync if module has no special options, otherwise if no module is found give help.
|
|
*)
|
|
for MODULE in ${MODULES:?}; do
|
|
if [[ "$1" == "$MODULE" ]]; then
|
|
eval sync_method="\${${MODULE}_sync_method:-rsync}"
|
|
if [[ "${sync_method:?}" == "git" ]]; then
|
|
git_sync "$@"
|
|
elif [[ "${sync_method:?}" == "aws" ]]; then
|
|
aws_sync "$@"
|
|
elif [[ "${sync_method:?}" == "ftp" ]]; then
|
|
ftp_sync "$@"
|
|
elif [[ "${sync_method:?}" == "wget" ]]; then
|
|
wget_sync "$@"
|
|
elif [[ "${sync_method:?}" == "qfm" ]]; then
|
|
quick_fedora_mirror_sync "$@"
|
|
else
|
|
rsync_sync "$@"
|
|
fi
|
|
exit 0
|
|
fi
|
|
done
|
|
# No module was found, so give help.
|
|
echo "Unknown module '$1'"
|
|
echo
|
|
print_help "$@"
|
|
;;
|
|
esac
|
|
done
|