From 21ab9fb1be5b4bd3f0464446dd805737bec4dad6 Mon Sep 17 00:00:00 2001
From: GRMrGecko <grmrgecko@gmail.com>
Date: Wed, 22 Nov 2023 15:57:22 -0600
Subject: [PATCH] Add s3cmd support.

---
 README.md      | 48 ++++++++++++++++++++++++++++++++++++++++++++----
 mirror-sync.sh | 50 +++++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 93 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index e41bc1d..404110b 100644
--- a/README.md
+++ b/README.md
@@ -99,7 +99,7 @@ Each module is configured via configurations prefixed by the module name. The on
 
 Each repo has at bare minimum the following configurations:
 
-- sync_method - rsync, git, aws, ftp, wget, or qfm.
+- sync_method - rsync, git, aws, s3cmd, ftp, wget, or qfm.
 - repo - The destination directory of the repository.
 - timestamp - Path to a file to store the last successful sync unix time stamp. Can be used by a monitoring system to confirm each repo is syncing successfully.
 
@@ -125,6 +125,9 @@ The bucket URL to sync with.
 #### aws_access_key
 The access key for the s3 bucket.
 
+### aws_secret_key
+The secret for the s3 bucket.
+
 #### aws_endpoint_url
 If you are using a third party S3 compatible service, you can enter their endpoint URL here.
 
@@ -141,6 +144,42 @@ example_aws_access_key="RANDOM_KEY_FROM_PROVIDER"
 example_aws_secret_key="RANDOM_SECRET_FROM_PROVIDER"
 ```
 
+### s3cmd
+Synchronize with an s3 bucket using s3cmd. To use this, you need the s3cmd package installed.
+
+#### aws_bucket
+The bucket URL to sync with.
+
+#### aws_access_key
+The access key for the s3 bucket.
+
+### aws_secret_key
+The secret for the s3 bucket.
+
+#### options
+Extra options to append to `aws s3 sync`.
+
+#### Example
+```bash
+example_sync_method="s3cmd"
+example_repo="/home/mirror/http/example"
+example_timestamp="/home/mirror/timestamp/example"
+example_aws_bucket="s3://bucket/directory"
+example_aws_access_key="RANDOM_KEY_FROM_PROVIDER"
+example_aws_secret_key="RANDOM_SECRET_FROM_PROVIDER"
+```
+
+Example of using third party bucket:
+```bash
+example_sync_method="s3cmd"
+example_repo="/home/mirror/http/example"
+example_timestamp="/home/mirror/timestamp/example"
+example_aws_bucket="s3://bucket/directory"
+example_aws_access_key="RANDOM_KEY_FROM_PROVIDER"
+example_aws_secret_key="RANDOM_SECRET_FROM_PROVIDER"
+example_options="--host='objects.example.com' --host-bucket='%(bucket).objects.example.com'"
+```
+
 ### ftp
 Synchronize both http and ftp sources to a repo. This sync method requires the lftp package to be installed.
 
@@ -332,6 +371,7 @@ There are not that many cli options available, usage is as follows:
 - sendmail
 - git
 - awscli
+- s3cmd
 - lftp
 - wget
 - rsync
@@ -340,15 +380,15 @@ There are not that many cli options available, usage is as follows:
 
 ### Install on RPM based servers
 ```bash
-yum install bash zsh sendmail git awscli lftp wget rsync
+yum install bash zsh sendmail git awscli s3cmd lftp wget rsync
 ```
 
 ### Install on DEB based servers
 ```bash
-apt install bash zsh sendmail git awscli lftp wget rsync
+apt install bash zsh sendmail git awscli s3cmd lftp wget rsync
 ```
 
 ### Install on Arch 
 ```bash
-yay -S bash zsh sendmail git aws-cli-git lftp wget rsync
+yay -S bash zsh sendmail git aws-cli-git s3cmd lftp wget rsync
 ```
diff --git a/mirror-sync.sh b/mirror-sync.sh
index 0a69393..bbb025a 100644
--- a/mirror-sync.sh
+++ b/mirror-sync.sh
@@ -5,7 +5,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/home/mirror/.
 
 # Variables for trace generation.
 PROGRAM="mirror-sync"
-VERSION="20231114"
+VERSION="20231122"
 TRACEHOST=$(hostname -f)
 mirror_hostname=$(hostname -f)
 DATE_STARTED=$(LC_ALL=POSIX LANG=POSIX date -u -R)
@@ -327,6 +327,52 @@ aws_sync() {
     log_end_header
 }
 
+# Sync AWS S3 bucket based mirrors using s3cmd.
+s3cmd_sync() {
+    MODULE=$1
+    acquire_lock "$MODULE"
+    
+    # Read the configuration for this module.
+    eval repo="\$${MODULE}_repo"
+    eval timestamp="\$${MODULE}_timestamp"
+    eval bucket="\$${MODULE}_aws_bucket"
+    eval AWS_ACCESS_KEY_ID="\$${MODULE}_aws_access_key"
+    eval AWS_SECRET_ACCESS_KEY="\$${MODULE}_aws_secret_key"
+    eval options="\$${MODULE}_options"
+    export AWS_ACCESS_KEY_ID
+    export AWS_SECRET_ACCESS_KEY
+
+    # If configuration is not set, exit.
+    if [[ ! $repo ]]; then
+        echo "No configuration exists for ${MODULE}"
+        exit 1
+    fi
+    log_start_header
+
+    # Run AWS client to sync the S3 bucket.
+    eval "$sync_timeout" s3cmd sync \
+        -v --progress \
+        --delete-after \
+        "$options" \
+        "'${bucket:?}'" "'${repo:?}'"
+    RT=${PIPESTATUS[0]}
+    if (( RT == 0 )); then
+        date +%s > "${timestamp:?}"
+        if [[ -e $ERRORFILE ]]; then
+            rm -f "$ERRORFILE"
+        fi
+    else
+        error_count=$((error_count+1))
+        if ((error_count>max_errors)); then
+            mail_error "Unable to sync with aws, check logs."
+            rm -f "$ERRORFILE"
+        fi
+        echo "$error_count" > "$ERRORFILE"
+    fi
+
+    log_end_header
+}
+
 # Sync using FTP.
 ftp_sync() {
     MODULE=$1
@@ -1002,6 +1048,8 @@ while (( $# > 0 )); do
                         git_sync "$@"
                     elif [[ "${sync_method:?}" == "aws" ]]; then
                         aws_sync "$@"
+                    elif [[ "${sync_method:?}" == "s3cmd" ]]; then
+                        s3cmd_sync "$@"
                     elif [[ "${sync_method:?}" == "ftp" ]]; then
                         ftp_sync "$@"
                     elif [[ "${sync_method:?}" == "wget" ]]; then