From 64fe7bd34f5463379556fb6424894c1eb793ad1c Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 7 Jan 2025 22:48:04 +0800 Subject: [PATCH] Ver 1.4 efficient lock management global lock mechanism Restic password handling --- changelogs.md | 32 ++++---- manifest.jps | 2 +- scripts/imports/backup_core_files.sh | 74 +++++++++++------- scripts/imports/backup_database.sh | 112 +++++++++++++++++++-------- scripts/imports/backup_media.sh | 75 ++++++++++++------ 5 files changed, 195 insertions(+), 100 deletions(-) diff --git a/changelogs.md b/changelogs.md index 12551cd..cc7599d 100644 --- a/changelogs.md +++ b/changelogs.md @@ -1,25 +1,27 @@ # Changelog -## Version 1.3 +## Version 1.4 ### Added -- Implemented Restic installation using a precompiled binary to reduce memory usage. -- Added logic to create the Restic password file if it doesn't exist during installation. -- Automatic installation of the cronnext tool for next-run calculation. +- Introduced efficient lock management in all backup scripts to prevent conflicts during simultaneous Restic operations. +- Automated removal of stale locks before each Restic operation. +- Added global lock mechanism using `flock` to serialize backup operations across multiple processes. +- Implemented dynamic inclusion of Restic password handling by reading it directly from the `/etc/restic-password` file. +- Added automatic validation of Restic repository access before performing backups. ### Fixed -- Resolved issue with missing Restic password file causing auto backup configuration to fail. -- Corrected permissions and ownership for the Restic password file to ensure it is accessible by the `litespeed` user. -- Updated the `installRestic` section in `manifest.jps` to generate a random password for the Restic password file during installation. -- Ensured consistent usage of the Restic password across all scripts by reading it from the `/etc/restic-password` file. -- Adjusted the `importScripts` section in `manifest.jps` to set the correct ownership for all backup-related directories to `litespeed:litespeed`. +- Fixed issues with manual Restic prompts for the repository password by ensuring passwords are passed via environment variables. +- Resolved potential conflicts caused by simultaneous Restic processes with the introduction of serialized operations. +- Corrected permissions for Restic lock directories to avoid permission-denied errors during backup and restore processes. +- Enhanced password validation logic to ensure backups fail gracefully if the provided password is incorrect. ### Updated -- Updated `manifest.jps` to ensure Restic password file creation and log rotation setup. +- Updated core, media, and database backup scripts to handle dynamic exclusion paths using a loop-based approach for `--exclude` options. +- Revised logging mechanisms across all scripts to include detailed timestamps and step-specific logs for better traceability. +- Improved script robustness by validating the Restic repository's accessibility upfront using `restic snapshots`. ### Improved -- Improved logging with timestamps and detailed error/warning levels. -- Enhanced fallback mechanism to manually calculate the next run time for common cron schedules. -- Implemented notifications via Slack and email for failures or issues. -- Optimized error handling to gracefully manage unsupported schedules or tool failures. - \ No newline at end of file +- Optimized all backup scripts to use efficient lock and unlock handling, ensuring smooth operation during concurrent backups and restores. +- Standardized backup script flow across media, database, and core backups, including consistent use of environment variables and error handling. +- Improved log formatting with more descriptive log messages and clear delineation of errors, warnings, and successes. +- Reduced redundancy in password handling by centralizing Restic password retrieval logic. \ No newline at end of file diff --git a/manifest.jps b/manifest.jps index 4a20b7d..561ed65 100644 --- a/manifest.jps +++ b/manifest.jps @@ -1,5 +1,5 @@ type: update -jpsVersion: 1.3 +jpsVersion: 1.4 name: MightyBox WordPress Backup/Restore Addon id: mb-backup-manager description: Custom Backup and Restore Addon for WordPress using Restic. Supports backing up databases, core files, media files, and full backups with scheduling and retention policies. diff --git a/scripts/imports/backup_core_files.sh b/scripts/imports/backup_core_files.sh index 362b13c..4fb2352 100644 --- a/scripts/imports/backup_core_files.sh +++ b/scripts/imports/backup_core_files.sh @@ -10,22 +10,6 @@ log_message() { echo "[$(date +'%Y-%m-%d %H:%M:%S')] $message" | tee -a "$LOG_FILE" } -# Function: Validate the password -validate_password() { - local password="$1" - if [ ! -f "$password_file" ]; then - log_message "ERROR: Password file not found at $password_file" - exit 1 - fi - - local stored_password - stored_password=$(<"$password_file") - if [ "$stored_password" != "$password" ]; then - log_message "ERROR: Password mismatch. Aborting backup." - exit 1 - fi -} - # Check for required arguments if [ $# -ne 2 ]; then echo "Usage: $0 " @@ -38,41 +22,73 @@ CUSTOM_LABEL="$2" # Configuration APP_PATH="/var/www/webroot/ROOT" -backupPath="/mnt/backups" -password_file="/etc/restic-password" +BACKUP_PATH="/mnt/backups" +PASSWORD_FILE="/etc/restic-password" LOG_DIR="/home/jelastic/mb-backups/logs" LOG_FILE="${LOG_DIR}/backup_core_files_$(date +'%Y-%m-%d').log" STATIC_TAG="core_files" # Static tag for this backup type -excludePaths=( +EXCLUDE_PATHS=( "$APP_PATH/wp-content/uploads" ) +LOCK_FILE="/tmp/restic_global.lock" # Ensure log directory exists mkdir -p "$LOG_DIR" -# Set Restic environment variables -export RESTIC_REPOSITORY="$backupPath" -export RESTIC_PASSWORD +# Validate password file and read password +if [ ! -f "$PASSWORD_FILE" ]; then + log_message "ERROR: Password file not found at $PASSWORD_FILE." + exit 1 +fi -# Validate the password -validate_password "$RESTIC_PASSWORD" +export RESTIC_PASSWORD=$(cat "$PASSWORD_FILE") + +# Verify repository access +log_message "Verifying repository access..." +if ! restic -r "$BACKUP_PATH" snapshots > /dev/null 2>&1; then + log_message "ERROR: Unable to access the Restic repository. Aborting backup." + exit 1 +fi + +# Acquire a global lock to serialize Restic operations +log_message "Acquiring global lock for Restic operations..." +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + log_message "Another Restic operation is running. Exiting." + exit 1 +fi +log_message "Global lock acquired." + +# Check and remove stale locks +log_message "Checking for stale locks in the repository..." +if restic -r "$BACKUP_PATH" list locks | grep -q "lock"; then + log_message "Stale locks detected. Unlocking the repository..." + restic -r "$BACKUP_PATH" unlock + log_message "Repository unlocked successfully." +else + log_message "No stale locks found." +fi # Logging start log_message "Starting Core Files Backup with tags: $STATIC_TAG, $CUSTOM_LABEL" # Build exclude options -excludeOptions="" -for path in "${excludePaths[@]}"; do - excludeOptions+="--exclude $path " +EXCLUDE_OPTIONS="" +for path in "${EXCLUDE_PATHS[@]}"; do + EXCLUDE_OPTIONS+="--exclude $path " done # Perform backup -if restic backup $excludeOptions "$APP_PATH" --tag "$STATIC_TAG" --tag "$CUSTOM_LABEL"; then +if restic -r "$BACKUP_PATH" backup $EXCLUDE_OPTIONS "$APP_PATH" \ + --tag "$STATIC_TAG" --tag "$CUSTOM_LABEL"; then log_message "Core files backup completed successfully." else log_message "ERROR: Core files backup failed." exit 1 fi +# Release global lock +exec 9>&- + # Logging end -log_message "Backup process finished." +log_message "Backup process finished successfully." diff --git a/scripts/imports/backup_database.sh b/scripts/imports/backup_database.sh index 3fcf694..1a84af0 100644 --- a/scripts/imports/backup_database.sh +++ b/scripts/imports/backup_database.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Exit immediately if a command fails +set -e + # Validate input parameters if [ "$#" -ne 2 ]; then echo "Usage: $0 " @@ -13,53 +16,100 @@ CUSTOM_TAG="$2" # Configuration APP_PATH='/var/www/webroot/ROOT' WP_CONFIG="${APP_PATH}/wp-config.php" -backupPath='/mnt/backups' -password_file="/etc/restic-password" +BACKUP_PATH='/mnt/backups' +PASSWORD_FILE="/etc/restic-password" LOG_DIR="/home/litespeed/mb-backups/logs" LOG_FILE="${LOG_DIR}/backup_database_$(date +'%Y-%m-%d').log" +LOCK_FILE="/tmp/restic_global.lock" + +# Ensure log directory exists +mkdir -p "$LOG_DIR" + +# Logging function +log() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" +} + +log "Starting Database Backup process with tags: wordpress_db, $CUSTOM_TAG." + +# Verify that the password file exists +if [ ! -f "$PASSWORD_FILE" ]; then + log "ERROR: Password file not found at $PASSWORD_FILE." + exit 1 +fi + +# Export the password from the file to ensure Restic uses it automatically +export RESTIC_PASSWORD=$(cat "$PASSWORD_FILE") + +# Verify backup path exists +if [ ! -d "$BACKUP_PATH" ]; then + log "ERROR: Backup path $BACKUP_PATH does not exist." + exit 1 +fi + +# Acquire a global lock to serialize Restic operations +log "Acquiring global lock for Restic operations..." +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + log "Another Restic operation is running. Exiting." + exit 1 +fi +log "Global lock acquired." + +# Check and remove stale locks +log "Checking for stale locks in the repository..." +if restic -r "$BACKUP_PATH" list locks | grep -q "lock"; then + log "Stale locks detected. Unlocking the repository..." + restic -r "$BACKUP_PATH" unlock + log "Repository unlocked successfully." +else + log "No stale locks found." +fi # Extract database credentials from wp-config.php +if [ ! -f "$WP_CONFIG" ]; then + log "ERROR: wp-config.php not found at $WP_CONFIG." + exit 1 +fi + DB_NAME=$(grep "define('DB_NAME'" "$WP_CONFIG" | cut -d "'" -f 4) DB_USER=$(grep "define('DB_USER'" "$WP_CONFIG" | cut -d "'" -f 4) DB_PASSWORD=$(grep "define('DB_PASSWORD'" "$WP_CONFIG" | cut -d "'" -f 4) DB_HOST=$(grep "define('DB_HOST'" "$WP_CONFIG" | cut -d "'" -f 4) DB_PORT=3306 # Default MySQL port -# Ensure log directory exists -mkdir -p "$LOG_DIR" +# Validate database credentials +if [ -z "$DB_NAME" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_HOST" ]; then + log "ERROR: Failed to extract database credentials from wp-config.php." + exit 1 +fi -# Set Restic and MySQL environment variables -export RESTIC_REPOSITORY="$backupPath" -export RESTIC_PASSWORD +# Set MySQL environment variable export MYSQL_PWD="$DB_PASSWORD" -# Logging start -echo "[$(date +'%Y-%m-%d %H:%M:%S')] Starting Database Backup for $DB_NAME with tags: wordpress_db, $CUSTOM_TAG" | tee -a "$LOG_FILE" - -# Verify that the password file exists -if [ ! -f "$password_file" ]; then - echo "ERROR: Password file not found at $password_file" | tee -a "$LOG_FILE" - exit 1 -fi - -# Verify backup path exists -if [ ! -d "$backupPath" ]; then - echo "ERROR: Backup path $backupPath does not exist." | tee -a "$LOG_FILE" - exit 1 -fi - -# Perform database backup with both static and custom tags +# Perform database backup with Restic BACKUP_TAGS="wordpress_db,$CUSTOM_TAG" -DUMP_FILE="${DB_NAME}_$(date +'%Y-%m-%d_%H-%M-%S').sql" +DUMP_FILE="/tmp/${DB_NAME}_$(date +'%Y-%m-%d_%H-%M-%S').sql" -if mysqldump -h "$DB_HOST" -P "$DB_PORT" -u "$DB_USER" "$DB_NAME" > "/tmp/$DUMP_FILE" && \ - restic backup --stdin --stdin-filename "$DUMP_FILE" --tag "$BACKUP_TAGS"; then - echo "[$(date +'%Y-%m-%d %H:%M:%S')] Database backup completed successfully with tags: $BACKUP_TAGS." | tee -a "$LOG_FILE" - rm -f "/tmp/$DUMP_FILE" +log "Performing database dump for $DB_NAME..." +if mysqldump -h "$DB_HOST" -P "$DB_PORT" -u "$DB_USER" "$DB_NAME" > "$DUMP_FILE"; then + log "Database dump created successfully: $DUMP_FILE" else - echo "[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: Database backup failed." | tee -a "$LOG_FILE" + log "ERROR: Database dump failed." exit 1 fi -# Logging end -echo "[$(date +'%Y-%m-%d %H:%M:%S')] Database Backup process finished." | tee -a "$LOG_FILE" +log "Backing up database dump to Restic repository with tags: $BACKUP_TAGS..." +if restic -r "$BACKUP_PATH" backup --stdin --stdin-filename "$(basename "$DUMP_FILE")" --tag "$BACKUP_TAGS" < "$DUMP_FILE"; then + log "Database backup completed successfully with tags: $BACKUP_TAGS." + rm -f "$DUMP_FILE" +else + log "ERROR: Restic backup failed." + rm -f "$DUMP_FILE" + exit 1 +fi + +# Release global lock +exec 9>&- + +log "Database Backup process finished successfully." diff --git a/scripts/imports/backup_media.sh b/scripts/imports/backup_media.sh index 464598d..2816f69 100644 --- a/scripts/imports/backup_media.sh +++ b/scripts/imports/backup_media.sh @@ -15,54 +15,81 @@ CUSTOM_TAG="$2" # Configuration APP_PATH='/var/www/webroot/ROOT' -backupPath='/mnt/backups' -password_file="/etc/restic-password" +BACKUP_PATH='/mnt/backups' +PASSWORD_FILE="/etc/restic-password" LOG_DIR="/home/jelastic/mb-backups/logs/manual/media" LOG_FILE="${LOG_DIR}/backup_media_$(date +'%Y-%m-%d').log" -includePaths=("$APP_PATH/wp-content/uploads") +INCLUDE_PATHS=("$APP_PATH/wp-content/uploads") +LOCK_FILE="/tmp/restic_global.lock" # Ensure log directory exists mkdir -p "$LOG_DIR" -# Check and fix permissions on /mnt/backups -if [ ! -w "$backupPath/locks" ]; then - echo "[$(date +'%Y-%m-%d %H:%M:%S')] Fixing permissions on $backupPath/locks" | tee -a "$LOG_FILE" - sudo chown -R litespeed:litespeed "$backupPath/locks" - sudo chmod -R u+rw "$backupPath/locks" -fi +# Logging function +log() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" +} -# Set Restic environment variables -export RESTIC_REPOSITORY="$backupPath" -export RESTIC_PASSWORD - -# Start logging -echo "[$(date +'%Y-%m-%d %H:%M:%S')] Starting Media Backup with tags: media_themes, $CUSTOM_TAG" | tee -a "$LOG_FILE" +log "Starting Media Backup process with tags: media_themes, $CUSTOM_TAG." # Verify password file exists -if [ ! -f "$password_file" ]; then - echo "ERROR: Password file not found at $password_file" | tee -a "$LOG_FILE" +if [ ! -f "$PASSWORD_FILE" ]; then + log "ERROR: Password file not found at $PASSWORD_FILE." exit 1 fi +# Export the password from the file to ensure Restic uses it automatically +export RESTIC_PASSWORD=$(cat "$PASSWORD_FILE") + # Verify repository access -if ! restic snapshots > /dev/null 2>&1; then - echo "ERROR: Unable to access the restic repository. Aborting backup." | tee -a "$LOG_FILE" +if ! restic -r "$BACKUP_PATH" snapshots > /dev/null 2>&1; then + log "ERROR: Unable to access the Restic repository. Aborting backup." exit 1 fi +# Acquire a global lock to serialize Restic operations +log "Acquiring global lock for Restic operations..." +exec 9>"$LOCK_FILE" +if ! flock -n 9; then + log "Another Restic operation is running. Exiting." + exit 1 +fi +log "Global lock acquired." + +# Check and remove stale locks +log "Checking for stale locks in the repository..." +if restic -r "$BACKUP_PATH" list locks | grep -q "lock"; then + log "Stale locks detected. Unlocking the repository..." + restic -r "$BACKUP_PATH" unlock + log "Repository unlocked successfully." +else + log "No stale locks found." +fi + +# Check and fix permissions on /mnt/backups +if [ ! -w "$BACKUP_PATH/locks" ]; then + log "Fixing permissions on $BACKUP_PATH/locks" + sudo chown -R litespeed:litespeed "$BACKUP_PATH/locks" + sudo chmod -R u+rw "$BACKUP_PATH/locks" +fi + # Perform the backup -for path in "${includePaths[@]}"; do - if restic backup "$path" \ +for path in "${INCLUDE_PATHS[@]}"; do + log "Starting backup for $path..." + if restic -r "$BACKUP_PATH" backup "$path" \ --tag media_themes \ --tag "$CUSTOM_TAG" \ --force \ --option b2.connections=4; then - echo "[$(date +'%Y-%m-%d %H:%M:%S')] Backup completed successfully for $path with tags: media_themes, $CUSTOM_TAG." | tee -a "$LOG_FILE" + log "Backup completed successfully for $path with tags: media_themes, $CUSTOM_TAG." else - echo "[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: Backup failed for $path." | tee -a "$LOG_FILE" + log "ERROR: Backup failed for $path." exit 1 fi done +# Release global lock +exec 9>&- + # End logging -echo "[$(date +'%Y-%m-%d %H:%M:%S')] Media Backup process finished." | tee -a "$LOG_FILE" +log "Media Backup process finished successfully."