Ver 1.4 efficient lock management global lock mechanism Restic password handling

changelogs.md

@@ -1,25 +1,27 @@
 # Changelog
 
-## Version 1.3
+## Version 1.4
 
 ### Added
-- Implemented Restic installation using a precompiled binary to reduce memory usage.
+- Introduced efficient lock management in all backup scripts to prevent conflicts during simultaneous Restic operations.
-- Added logic to create the Restic password file if it doesn't exist during installation.
+- Automated removal of stale locks before each Restic operation.
-- Automatic installation of the cronnext tool for next-run calculation.
+- Added global lock mechanism using `flock` to serialize backup operations across multiple processes.
+- Implemented dynamic inclusion of Restic password handling by reading it directly from the `/etc/restic-password` file.
+- Added automatic validation of Restic repository access before performing backups.
 
 ### Fixed
-- Resolved issue with missing Restic password file causing auto backup configuration to fail.
+- Fixed issues with manual Restic prompts for the repository password by ensuring passwords are passed via environment variables.
-- Corrected permissions and ownership for the Restic password file to ensure it is accessible by the `litespeed` user.
+- Resolved potential conflicts caused by simultaneous Restic processes with the introduction of serialized operations.
-- Updated the `installRestic` section in `manifest.jps` to generate a random password for the Restic password file during installation.
+- Corrected permissions for Restic lock directories to avoid permission-denied errors during backup and restore processes.
-- Ensured consistent usage of the Restic password across all scripts by reading it from the `/etc/restic-password` file.
+- Enhanced password validation logic to ensure backups fail gracefully if the provided password is incorrect.
-- Adjusted the `importScripts` section in `manifest.jps` to set the correct ownership for all backup-related directories to `litespeed:litespeed`.
 
 ### Updated
-- Updated `manifest.jps` to ensure Restic password file creation and log rotation setup.
+- Updated core, media, and database backup scripts to handle dynamic exclusion paths using a loop-based approach for `--exclude` options.
+- Revised logging mechanisms across all scripts to include detailed timestamps and step-specific logs for better traceability.
+- Improved script robustness by validating the Restic repository's accessibility upfront using `restic snapshots`.
 
 ### Improved
-- Improved logging with timestamps and detailed error/warning levels.
+- Optimized all backup scripts to use efficient lock and unlock handling, ensuring smooth operation during concurrent backups and restores.
-- Enhanced fallback mechanism to manually calculate the next run time for common cron schedules.
+- Standardized backup script flow across media, database, and core backups, including consistent use of environment variables and error handling.
-- Implemented notifications via Slack and email for failures or issues.
+- Improved log formatting with more descriptive log messages and clear delineation of errors, warnings, and successes.
-- Optimized error handling to gracefully manage unsupported schedules or tool failures.
+- Reduced redundancy in password handling by centralizing Restic password retrieval logic.
-     

manifest.jps

@@ -1,5 +1,5 @@
 type: update
-jpsVersion: 1.3
+jpsVersion: 1.4
 name: MightyBox WordPress Backup/Restore Addon
 id: mb-backup-manager
 description: Custom Backup and Restore Addon for WordPress using Restic. Supports backing up databases, core files, media files, and full backups with scheduling and retention policies.

scripts/imports/backup_core_files.sh

@@ -10,22 +10,6 @@ log_message() {
     echo "[$(date +'%Y-%m-%d %H:%M:%S')] $message" | tee -a "$LOG_FILE"
 }
 
-# Function: Validate the password
-validate_password() {
-    local password="$1"
-    if [ ! -f "$password_file" ]; then
-        log_message "ERROR: Password file not found at $password_file"
-        exit 1
-    fi
-
-    local stored_password
-    stored_password=$(<"$password_file")
-    if [ "$stored_password" != "$password" ]; then
-        log_message "ERROR: Password mismatch. Aborting backup."
-        exit 1
-    fi
-}
-
 # Check for required arguments
 if [ $# -ne 2 ]; then
     echo "Usage: $0 <RESTIC_PASSWORD> <CUSTOM_LABEL>"
@@ -38,41 +22,73 @@ CUSTOM_LABEL="$2"
 
 # Configuration
 APP_PATH="/var/www/webroot/ROOT"
-backupPath="/mnt/backups"
+BACKUP_PATH="/mnt/backups"
-password_file="/etc/restic-password"
+PASSWORD_FILE="/etc/restic-password"
 LOG_DIR="/home/jelastic/mb-backups/logs"
 LOG_FILE="${LOG_DIR}/backup_core_files_$(date +'%Y-%m-%d').log"
 STATIC_TAG="core_files"  # Static tag for this backup type
-excludePaths=(
+EXCLUDE_PATHS=(
     "$APP_PATH/wp-content/uploads"
 )
+LOCK_FILE="/tmp/restic_global.lock"
 
 # Ensure log directory exists
 mkdir -p "$LOG_DIR"
 
-# Set Restic environment variables
+# Validate password file and read password
-export RESTIC_REPOSITORY="$backupPath"
+if [ ! -f "$PASSWORD_FILE" ]; then
-export RESTIC_PASSWORD
+    log_message "ERROR: Password file not found at $PASSWORD_FILE."
+    exit 1
+fi
 
-# Validate the password
+export RESTIC_PASSWORD=$(cat "$PASSWORD_FILE")
-validate_password "$RESTIC_PASSWORD"
+
+# Verify repository access
+log_message "Verifying repository access..."
+if ! restic -r "$BACKUP_PATH" snapshots > /dev/null 2>&1; then
+    log_message "ERROR: Unable to access the Restic repository. Aborting backup."
+    exit 1
+fi
+
+# Acquire a global lock to serialize Restic operations
+log_message "Acquiring global lock for Restic operations..."
+exec 9>"$LOCK_FILE"
+if ! flock -n 9; then
+    log_message "Another Restic operation is running. Exiting."
+    exit 1
+fi
+log_message "Global lock acquired."
+
+# Check and remove stale locks
+log_message "Checking for stale locks in the repository..."
+if restic -r "$BACKUP_PATH" list locks | grep -q "lock"; then
+    log_message "Stale locks detected. Unlocking the repository..."
+    restic -r "$BACKUP_PATH" unlock
+    log_message "Repository unlocked successfully."
+else
+    log_message "No stale locks found."
+fi
 
 # Logging start
 log_message "Starting Core Files Backup with tags: $STATIC_TAG, $CUSTOM_LABEL"
 
 # Build exclude options
-excludeOptions=""
+EXCLUDE_OPTIONS=""
-for path in "${excludePaths[@]}"; do
+for path in "${EXCLUDE_PATHS[@]}"; do
-    excludeOptions+="--exclude $path "
+    EXCLUDE_OPTIONS+="--exclude $path "
 done
 
 # Perform backup
-if restic backup $excludeOptions "$APP_PATH" --tag "$STATIC_TAG" --tag "$CUSTOM_LABEL"; then
+if restic -r "$BACKUP_PATH" backup $EXCLUDE_OPTIONS "$APP_PATH" \
+    --tag "$STATIC_TAG" --tag "$CUSTOM_LABEL"; then
     log_message "Core files backup completed successfully."
 else
     log_message "ERROR: Core files backup failed."
     exit 1
 fi
 
+# Release global lock
+exec 9>&-
+
 # Logging end
-log_message "Backup process finished."
+log_message "Backup process finished successfully."

scripts/imports/backup_database.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Exit immediately if a command fails
+set -e
+
 # Validate input parameters
 if [ "$#" -ne 2 ]; then
     echo "Usage: $0 <RESTIC_PASSWORD> <ADDITIONAL_TAG>"
@@ -13,53 +16,100 @@ CUSTOM_TAG="$2"
 # Configuration
 APP_PATH='/var/www/webroot/ROOT'
 WP_CONFIG="${APP_PATH}/wp-config.php"
-backupPath='/mnt/backups'
+BACKUP_PATH='/mnt/backups'
-password_file="/etc/restic-password"
+PASSWORD_FILE="/etc/restic-password"
 LOG_DIR="/home/litespeed/mb-backups/logs"
 LOG_FILE="${LOG_DIR}/backup_database_$(date +'%Y-%m-%d').log"
+LOCK_FILE="/tmp/restic_global.lock"
+
+# Ensure log directory exists
+mkdir -p "$LOG_DIR"
+
+# Logging function
+log() {
+    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
+}
+
+log "Starting Database Backup process with tags: wordpress_db, $CUSTOM_TAG."
+
+# Verify that the password file exists
+if [ ! -f "$PASSWORD_FILE" ]; then
+    log "ERROR: Password file not found at $PASSWORD_FILE."
+    exit 1
+fi
+
+# Export the password from the file to ensure Restic uses it automatically
+export RESTIC_PASSWORD=$(cat "$PASSWORD_FILE")
+
+# Verify backup path exists
+if [ ! -d "$BACKUP_PATH" ]; then
+    log "ERROR: Backup path $BACKUP_PATH does not exist."
+    exit 1
+fi
+
+# Acquire a global lock to serialize Restic operations
+log "Acquiring global lock for Restic operations..."
+exec 9>"$LOCK_FILE"
+if ! flock -n 9; then
+    log "Another Restic operation is running. Exiting."
+    exit 1
+fi
+log "Global lock acquired."
+
+# Check and remove stale locks
+log "Checking for stale locks in the repository..."
+if restic -r "$BACKUP_PATH" list locks | grep -q "lock"; then
+    log "Stale locks detected. Unlocking the repository..."
+    restic -r "$BACKUP_PATH" unlock
+    log "Repository unlocked successfully."
+else
+    log "No stale locks found."
+fi
 
 # Extract database credentials from wp-config.php
+if [ ! -f "$WP_CONFIG" ]; then
+    log "ERROR: wp-config.php not found at $WP_CONFIG."
+    exit 1
+fi
+
 DB_NAME=$(grep "define('DB_NAME'" "$WP_CONFIG" | cut -d "'" -f 4)
 DB_USER=$(grep "define('DB_USER'" "$WP_CONFIG" | cut -d "'" -f 4)
 DB_PASSWORD=$(grep "define('DB_PASSWORD'" "$WP_CONFIG" | cut -d "'" -f 4)
 DB_HOST=$(grep "define('DB_HOST'" "$WP_CONFIG" | cut -d "'" -f 4)
 DB_PORT=3306  # Default MySQL port
 
-# Ensure log directory exists
+# Validate database credentials
-mkdir -p "$LOG_DIR"
+if [ -z "$DB_NAME" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_HOST" ]; then
+    log "ERROR: Failed to extract database credentials from wp-config.php."
+    exit 1
+fi
 
-# Set Restic and MySQL environment variables
+# Set MySQL environment variable
-export RESTIC_REPOSITORY="$backupPath"
-export RESTIC_PASSWORD
 export MYSQL_PWD="$DB_PASSWORD"
 
-# Logging start
+# Perform database backup with Restic
-echo "[$(date +'%Y-%m-%d %H:%M:%S')] Starting Database Backup for $DB_NAME with tags: wordpress_db, $CUSTOM_TAG" | tee -a "$LOG_FILE"
-
-# Verify that the password file exists
-if [ ! -f "$password_file" ]; then
-    echo "ERROR: Password file not found at $password_file" | tee -a "$LOG_FILE"
-    exit 1
-fi
-
-# Verify backup path exists
-if [ ! -d "$backupPath" ]; then
-    echo "ERROR: Backup path $backupPath does not exist." | tee -a "$LOG_FILE"
-    exit 1
-fi
-
-# Perform database backup with both static and custom tags
 BACKUP_TAGS="wordpress_db,$CUSTOM_TAG"
-DUMP_FILE="${DB_NAME}_$(date +'%Y-%m-%d_%H-%M-%S').sql"
+DUMP_FILE="/tmp/${DB_NAME}_$(date +'%Y-%m-%d_%H-%M-%S').sql"
 
-if mysqldump -h "$DB_HOST" -P "$DB_PORT" -u "$DB_USER" "$DB_NAME" > "/tmp/$DUMP_FILE" && \
+log "Performing database dump for $DB_NAME..."
-   restic backup --stdin --stdin-filename "$DUMP_FILE" --tag "$BACKUP_TAGS"; then
+if mysqldump -h "$DB_HOST" -P "$DB_PORT" -u "$DB_USER" "$DB_NAME" > "$DUMP_FILE"; then
-    echo "[$(date +'%Y-%m-%d %H:%M:%S')] Database backup completed successfully with tags: $BACKUP_TAGS." | tee -a "$LOG_FILE"
+    log "Database dump created successfully: $DUMP_FILE"
-    rm -f "/tmp/$DUMP_FILE"
 else
-    echo "[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: Database backup failed." | tee -a "$LOG_FILE"
+    log "ERROR: Database dump failed."
     exit 1
 fi
 
-# Logging end
+log "Backing up database dump to Restic repository with tags: $BACKUP_TAGS..."
-echo "[$(date +'%Y-%m-%d %H:%M:%S')] Database Backup process finished." | tee -a "$LOG_FILE"
+if restic -r "$BACKUP_PATH" backup --stdin --stdin-filename "$(basename "$DUMP_FILE")" --tag "$BACKUP_TAGS" < "$DUMP_FILE"; then
+    log "Database backup completed successfully with tags: $BACKUP_TAGS."
+    rm -f "$DUMP_FILE"
+else
+    log "ERROR: Restic backup failed."
+    rm -f "$DUMP_FILE"
+    exit 1
+fi
+
+# Release global lock
+exec 9>&-
+
+log "Database Backup process finished successfully."

scripts/imports/backup_media.sh

@@ -15,54 +15,81 @@ CUSTOM_TAG="$2"
 
 # Configuration
 APP_PATH='/var/www/webroot/ROOT'
-backupPath='/mnt/backups'
+BACKUP_PATH='/mnt/backups'
-password_file="/etc/restic-password"
+PASSWORD_FILE="/etc/restic-password"
 LOG_DIR="/home/jelastic/mb-backups/logs/manual/media"
 LOG_FILE="${LOG_DIR}/backup_media_$(date +'%Y-%m-%d').log"
-includePaths=("$APP_PATH/wp-content/uploads")
+INCLUDE_PATHS=("$APP_PATH/wp-content/uploads")
+LOCK_FILE="/tmp/restic_global.lock"
 
 # Ensure log directory exists
 mkdir -p "$LOG_DIR"
 
-# Check and fix permissions on /mnt/backups
+# Logging function
-if [ ! -w "$backupPath/locks" ]; then
+log() {
-    echo "[$(date +'%Y-%m-%d %H:%M:%S')] Fixing permissions on $backupPath/locks" | tee -a "$LOG_FILE"
+    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
-    sudo chown -R litespeed:litespeed "$backupPath/locks"
+}
-    sudo chmod -R u+rw "$backupPath/locks"
-fi
 
-# Set Restic environment variables
+log "Starting Media Backup process with tags: media_themes, $CUSTOM_TAG."
-export RESTIC_REPOSITORY="$backupPath"
-export RESTIC_PASSWORD
-
-# Start logging
-echo "[$(date +'%Y-%m-%d %H:%M:%S')] Starting Media Backup with tags: media_themes, $CUSTOM_TAG" | tee -a "$LOG_FILE"
 
 # Verify password file exists
-if [ ! -f "$password_file" ]; then
+if [ ! -f "$PASSWORD_FILE" ]; then
-    echo "ERROR: Password file not found at $password_file" | tee -a "$LOG_FILE"
+    log "ERROR: Password file not found at $PASSWORD_FILE."
     exit 1
 fi
 
+# Export the password from the file to ensure Restic uses it automatically
+export RESTIC_PASSWORD=$(cat "$PASSWORD_FILE")
+
 # Verify repository access
-if ! restic snapshots > /dev/null 2>&1; then
+if ! restic -r "$BACKUP_PATH" snapshots > /dev/null 2>&1; then
-    echo "ERROR: Unable to access the restic repository. Aborting backup." | tee -a "$LOG_FILE"
+    log "ERROR: Unable to access the Restic repository. Aborting backup."
     exit 1
 fi
 
+# Acquire a global lock to serialize Restic operations
+log "Acquiring global lock for Restic operations..."
+exec 9>"$LOCK_FILE"
+if ! flock -n 9; then
+    log "Another Restic operation is running. Exiting."
+    exit 1
+fi
+log "Global lock acquired."
+
+# Check and remove stale locks
+log "Checking for stale locks in the repository..."
+if restic -r "$BACKUP_PATH" list locks | grep -q "lock"; then
+    log "Stale locks detected. Unlocking the repository..."
+    restic -r "$BACKUP_PATH" unlock
+    log "Repository unlocked successfully."
+else
+    log "No stale locks found."
+fi
+
+# Check and fix permissions on /mnt/backups
+if [ ! -w "$BACKUP_PATH/locks" ]; then
+    log "Fixing permissions on $BACKUP_PATH/locks"
+    sudo chown -R litespeed:litespeed "$BACKUP_PATH/locks"
+    sudo chmod -R u+rw "$BACKUP_PATH/locks"
+fi
+
 # Perform the backup
-for path in "${includePaths[@]}"; do
+for path in "${INCLUDE_PATHS[@]}"; do
-    if restic backup "$path" \
+    log "Starting backup for $path..."
+    if restic -r "$BACKUP_PATH" backup "$path" \
         --tag media_themes \
         --tag "$CUSTOM_TAG" \
         --force \
         --option b2.connections=4; then
-        echo "[$(date +'%Y-%m-%d %H:%M:%S')] Backup completed successfully for $path with tags: media_themes, $CUSTOM_TAG." | tee -a "$LOG_FILE"
+        log "Backup completed successfully for $path with tags: media_themes, $CUSTOM_TAG."
     else
-        echo "[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: Backup failed for $path." | tee -a "$LOG_FILE"
+        log "ERROR: Backup failed for $path."
         exit 1
     fi
 done
 
+# Release global lock
+exec 9>&-
+
 # End logging
-echo "[$(date +'%Y-%m-%d %H:%M:%S')] Media Backup process finished." | tee -a "$LOG_FILE"
+log "Media Backup process finished successfully."