Ver 1.4 efficient lock management global lock mechanism Restic password handling
parent
2965665f38
commit
64fe7bd34f
|
|
@ -1,25 +1,27 @@
|
|||
# Changelog
|
||||
|
||||
## Version 1.3
|
||||
## Version 1.4
|
||||
|
||||
### Added
|
||||
- Implemented Restic installation using a precompiled binary to reduce memory usage.
|
||||
- Added logic to create the Restic password file if it doesn't exist during installation.
|
||||
- Automatic installation of the cronnext tool for next-run calculation.
|
||||
- Introduced efficient lock management in all backup scripts to prevent conflicts during simultaneous Restic operations.
|
||||
- Automated removal of stale locks before each Restic operation.
|
||||
- Added global lock mechanism using `flock` to serialize backup operations across multiple processes.
|
||||
- Implemented dynamic inclusion of Restic password handling by reading it directly from the `/etc/restic-password` file.
|
||||
- Added automatic validation of Restic repository access before performing backups.
|
||||
|
||||
### Fixed
|
||||
- Resolved issue with missing Restic password file causing auto backup configuration to fail.
|
||||
- Corrected permissions and ownership for the Restic password file to ensure it is accessible by the `litespeed` user.
|
||||
- Updated the `installRestic` section in `manifest.jps` to generate a random password for the Restic password file during installation.
|
||||
- Ensured consistent usage of the Restic password across all scripts by reading it from the `/etc/restic-password` file.
|
||||
- Adjusted the `importScripts` section in `manifest.jps` to set the correct ownership for all backup-related directories to `litespeed:litespeed`.
|
||||
- Fixed issues with manual Restic prompts for the repository password by ensuring passwords are passed via environment variables.
|
||||
- Resolved potential conflicts caused by simultaneous Restic processes with the introduction of serialized operations.
|
||||
- Corrected permissions for Restic lock directories to avoid permission-denied errors during backup and restore processes.
|
||||
- Enhanced password validation logic to ensure backups fail gracefully if the provided password is incorrect.
|
||||
|
||||
### Updated
|
||||
- Updated `manifest.jps` to ensure Restic password file creation and log rotation setup.
|
||||
- Updated core, media, and database backup scripts to handle dynamic exclusion paths using a loop-based approach for `--exclude` options.
|
||||
- Revised logging mechanisms across all scripts to include detailed timestamps and step-specific logs for better traceability.
|
||||
- Improved script robustness by validating the Restic repository's accessibility upfront using `restic snapshots`.
|
||||
|
||||
### Improved
|
||||
- Improved logging with timestamps and detailed error/warning levels.
|
||||
- Enhanced fallback mechanism to manually calculate the next run time for common cron schedules.
|
||||
- Implemented notifications via Slack and email for failures or issues.
|
||||
- Optimized error handling to gracefully manage unsupported schedules or tool failures.
|
||||
|
||||
- Optimized all backup scripts to use efficient lock and unlock handling, ensuring smooth operation during concurrent backups and restores.
|
||||
- Standardized backup script flow across media, database, and core backups, including consistent use of environment variables and error handling.
|
||||
- Improved log formatting with more descriptive log messages and clear delineation of errors, warnings, and successes.
|
||||
- Reduced redundancy in password handling by centralizing Restic password retrieval logic.
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
type: update
|
||||
jpsVersion: 1.3
|
||||
jpsVersion: 1.4
|
||||
name: MightyBox WordPress Backup/Restore Addon
|
||||
id: mb-backup-manager
|
||||
description: Custom Backup and Restore Addon for WordPress using Restic. Supports backing up databases, core files, media files, and full backups with scheduling and retention policies.
|
||||
|
|
|
|||
|
|
@ -10,22 +10,6 @@ log_message() {
|
|||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] $message" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Function: Validate the password
|
||||
validate_password() {
|
||||
local password="$1"
|
||||
if [ ! -f "$password_file" ]; then
|
||||
log_message "ERROR: Password file not found at $password_file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local stored_password
|
||||
stored_password=$(<"$password_file")
|
||||
if [ "$stored_password" != "$password" ]; then
|
||||
log_message "ERROR: Password mismatch. Aborting backup."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check for required arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "Usage: $0 <RESTIC_PASSWORD> <CUSTOM_LABEL>"
|
||||
|
|
@ -38,41 +22,73 @@ CUSTOM_LABEL="$2"
|
|||
|
||||
# Configuration
|
||||
APP_PATH="/var/www/webroot/ROOT"
|
||||
backupPath="/mnt/backups"
|
||||
password_file="/etc/restic-password"
|
||||
BACKUP_PATH="/mnt/backups"
|
||||
PASSWORD_FILE="/etc/restic-password"
|
||||
LOG_DIR="/home/jelastic/mb-backups/logs"
|
||||
LOG_FILE="${LOG_DIR}/backup_core_files_$(date +'%Y-%m-%d').log"
|
||||
STATIC_TAG="core_files" # Static tag for this backup type
|
||||
excludePaths=(
|
||||
EXCLUDE_PATHS=(
|
||||
"$APP_PATH/wp-content/uploads"
|
||||
)
|
||||
LOCK_FILE="/tmp/restic_global.lock"
|
||||
|
||||
# Ensure log directory exists
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Set Restic environment variables
|
||||
export RESTIC_REPOSITORY="$backupPath"
|
||||
export RESTIC_PASSWORD
|
||||
# Validate password file and read password
|
||||
if [ ! -f "$PASSWORD_FILE" ]; then
|
||||
log_message "ERROR: Password file not found at $PASSWORD_FILE."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Validate the password
|
||||
validate_password "$RESTIC_PASSWORD"
|
||||
export RESTIC_PASSWORD=$(cat "$PASSWORD_FILE")
|
||||
|
||||
# Verify repository access
|
||||
log_message "Verifying repository access..."
|
||||
if ! restic -r "$BACKUP_PATH" snapshots > /dev/null 2>&1; then
|
||||
log_message "ERROR: Unable to access the Restic repository. Aborting backup."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Acquire a global lock to serialize Restic operations
|
||||
log_message "Acquiring global lock for Restic operations..."
|
||||
exec 9>"$LOCK_FILE"
|
||||
if ! flock -n 9; then
|
||||
log_message "Another Restic operation is running. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
log_message "Global lock acquired."
|
||||
|
||||
# Check and remove stale locks
|
||||
log_message "Checking for stale locks in the repository..."
|
||||
if restic -r "$BACKUP_PATH" list locks | grep -q "lock"; then
|
||||
log_message "Stale locks detected. Unlocking the repository..."
|
||||
restic -r "$BACKUP_PATH" unlock
|
||||
log_message "Repository unlocked successfully."
|
||||
else
|
||||
log_message "No stale locks found."
|
||||
fi
|
||||
|
||||
# Logging start
|
||||
log_message "Starting Core Files Backup with tags: $STATIC_TAG, $CUSTOM_LABEL"
|
||||
|
||||
# Build exclude options
|
||||
excludeOptions=""
|
||||
for path in "${excludePaths[@]}"; do
|
||||
excludeOptions+="--exclude $path "
|
||||
EXCLUDE_OPTIONS=""
|
||||
for path in "${EXCLUDE_PATHS[@]}"; do
|
||||
EXCLUDE_OPTIONS+="--exclude $path "
|
||||
done
|
||||
|
||||
# Perform backup
|
||||
if restic backup $excludeOptions "$APP_PATH" --tag "$STATIC_TAG" --tag "$CUSTOM_LABEL"; then
|
||||
if restic -r "$BACKUP_PATH" backup $EXCLUDE_OPTIONS "$APP_PATH" \
|
||||
--tag "$STATIC_TAG" --tag "$CUSTOM_LABEL"; then
|
||||
log_message "Core files backup completed successfully."
|
||||
else
|
||||
log_message "ERROR: Core files backup failed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Release global lock
|
||||
exec 9>&-
|
||||
|
||||
# Logging end
|
||||
log_message "Backup process finished."
|
||||
log_message "Backup process finished successfully."
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Exit immediately if a command fails
|
||||
set -e
|
||||
|
||||
# Validate input parameters
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "Usage: $0 <RESTIC_PASSWORD> <ADDITIONAL_TAG>"
|
||||
|
|
@ -13,53 +16,100 @@ CUSTOM_TAG="$2"
|
|||
# Configuration
|
||||
APP_PATH='/var/www/webroot/ROOT'
|
||||
WP_CONFIG="${APP_PATH}/wp-config.php"
|
||||
backupPath='/mnt/backups'
|
||||
password_file="/etc/restic-password"
|
||||
BACKUP_PATH='/mnt/backups'
|
||||
PASSWORD_FILE="/etc/restic-password"
|
||||
LOG_DIR="/home/litespeed/mb-backups/logs"
|
||||
LOG_FILE="${LOG_DIR}/backup_database_$(date +'%Y-%m-%d').log"
|
||||
LOCK_FILE="/tmp/restic_global.lock"
|
||||
|
||||
# Ensure log directory exists
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Logging function
|
||||
log() {
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
log "Starting Database Backup process with tags: wordpress_db, $CUSTOM_TAG."
|
||||
|
||||
# Verify that the password file exists
|
||||
if [ ! -f "$PASSWORD_FILE" ]; then
|
||||
log "ERROR: Password file not found at $PASSWORD_FILE."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export the password from the file to ensure Restic uses it automatically
|
||||
export RESTIC_PASSWORD=$(cat "$PASSWORD_FILE")
|
||||
|
||||
# Verify backup path exists
|
||||
if [ ! -d "$BACKUP_PATH" ]; then
|
||||
log "ERROR: Backup path $BACKUP_PATH does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Acquire a global lock to serialize Restic operations
|
||||
log "Acquiring global lock for Restic operations..."
|
||||
exec 9>"$LOCK_FILE"
|
||||
if ! flock -n 9; then
|
||||
log "Another Restic operation is running. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
log "Global lock acquired."
|
||||
|
||||
# Check and remove stale locks
|
||||
log "Checking for stale locks in the repository..."
|
||||
if restic -r "$BACKUP_PATH" list locks | grep -q "lock"; then
|
||||
log "Stale locks detected. Unlocking the repository..."
|
||||
restic -r "$BACKUP_PATH" unlock
|
||||
log "Repository unlocked successfully."
|
||||
else
|
||||
log "No stale locks found."
|
||||
fi
|
||||
|
||||
# Extract database credentials from wp-config.php
|
||||
if [ ! -f "$WP_CONFIG" ]; then
|
||||
log "ERROR: wp-config.php not found at $WP_CONFIG."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DB_NAME=$(grep "define('DB_NAME'" "$WP_CONFIG" | cut -d "'" -f 4)
|
||||
DB_USER=$(grep "define('DB_USER'" "$WP_CONFIG" | cut -d "'" -f 4)
|
||||
DB_PASSWORD=$(grep "define('DB_PASSWORD'" "$WP_CONFIG" | cut -d "'" -f 4)
|
||||
DB_HOST=$(grep "define('DB_HOST'" "$WP_CONFIG" | cut -d "'" -f 4)
|
||||
DB_PORT=3306 # Default MySQL port
|
||||
|
||||
# Ensure log directory exists
|
||||
mkdir -p "$LOG_DIR"
|
||||
# Validate database credentials
|
||||
if [ -z "$DB_NAME" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_HOST" ]; then
|
||||
log "ERROR: Failed to extract database credentials from wp-config.php."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Set Restic and MySQL environment variables
|
||||
export RESTIC_REPOSITORY="$backupPath"
|
||||
export RESTIC_PASSWORD
|
||||
# Set MySQL environment variable
|
||||
export MYSQL_PWD="$DB_PASSWORD"
|
||||
|
||||
# Logging start
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] Starting Database Backup for $DB_NAME with tags: wordpress_db, $CUSTOM_TAG" | tee -a "$LOG_FILE"
|
||||
|
||||
# Verify that the password file exists
|
||||
if [ ! -f "$password_file" ]; then
|
||||
echo "ERROR: Password file not found at $password_file" | tee -a "$LOG_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify backup path exists
|
||||
if [ ! -d "$backupPath" ]; then
|
||||
echo "ERROR: Backup path $backupPath does not exist." | tee -a "$LOG_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Perform database backup with both static and custom tags
|
||||
# Perform database backup with Restic
|
||||
BACKUP_TAGS="wordpress_db,$CUSTOM_TAG"
|
||||
DUMP_FILE="${DB_NAME}_$(date +'%Y-%m-%d_%H-%M-%S').sql"
|
||||
DUMP_FILE="/tmp/${DB_NAME}_$(date +'%Y-%m-%d_%H-%M-%S').sql"
|
||||
|
||||
if mysqldump -h "$DB_HOST" -P "$DB_PORT" -u "$DB_USER" "$DB_NAME" > "/tmp/$DUMP_FILE" && \
|
||||
restic backup --stdin --stdin-filename "$DUMP_FILE" --tag "$BACKUP_TAGS"; then
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] Database backup completed successfully with tags: $BACKUP_TAGS." | tee -a "$LOG_FILE"
|
||||
rm -f "/tmp/$DUMP_FILE"
|
||||
log "Performing database dump for $DB_NAME..."
|
||||
if mysqldump -h "$DB_HOST" -P "$DB_PORT" -u "$DB_USER" "$DB_NAME" > "$DUMP_FILE"; then
|
||||
log "Database dump created successfully: $DUMP_FILE"
|
||||
else
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: Database backup failed." | tee -a "$LOG_FILE"
|
||||
log "ERROR: Database dump failed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Logging end
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] Database Backup process finished." | tee -a "$LOG_FILE"
|
||||
log "Backing up database dump to Restic repository with tags: $BACKUP_TAGS..."
|
||||
if restic -r "$BACKUP_PATH" backup --stdin --stdin-filename "$(basename "$DUMP_FILE")" --tag "$BACKUP_TAGS" < "$DUMP_FILE"; then
|
||||
log "Database backup completed successfully with tags: $BACKUP_TAGS."
|
||||
rm -f "$DUMP_FILE"
|
||||
else
|
||||
log "ERROR: Restic backup failed."
|
||||
rm -f "$DUMP_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Release global lock
|
||||
exec 9>&-
|
||||
|
||||
log "Database Backup process finished successfully."
|
||||
|
|
|
|||
|
|
@ -15,54 +15,81 @@ CUSTOM_TAG="$2"
|
|||
|
||||
# Configuration
|
||||
APP_PATH='/var/www/webroot/ROOT'
|
||||
backupPath='/mnt/backups'
|
||||
password_file="/etc/restic-password"
|
||||
BACKUP_PATH='/mnt/backups'
|
||||
PASSWORD_FILE="/etc/restic-password"
|
||||
LOG_DIR="/home/jelastic/mb-backups/logs/manual/media"
|
||||
LOG_FILE="${LOG_DIR}/backup_media_$(date +'%Y-%m-%d').log"
|
||||
includePaths=("$APP_PATH/wp-content/uploads")
|
||||
INCLUDE_PATHS=("$APP_PATH/wp-content/uploads")
|
||||
LOCK_FILE="/tmp/restic_global.lock"
|
||||
|
||||
# Ensure log directory exists
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Check and fix permissions on /mnt/backups
|
||||
if [ ! -w "$backupPath/locks" ]; then
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] Fixing permissions on $backupPath/locks" | tee -a "$LOG_FILE"
|
||||
sudo chown -R litespeed:litespeed "$backupPath/locks"
|
||||
sudo chmod -R u+rw "$backupPath/locks"
|
||||
fi
|
||||
# Logging function
|
||||
log() {
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Set Restic environment variables
|
||||
export RESTIC_REPOSITORY="$backupPath"
|
||||
export RESTIC_PASSWORD
|
||||
|
||||
# Start logging
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] Starting Media Backup with tags: media_themes, $CUSTOM_TAG" | tee -a "$LOG_FILE"
|
||||
log "Starting Media Backup process with tags: media_themes, $CUSTOM_TAG."
|
||||
|
||||
# Verify password file exists
|
||||
if [ ! -f "$password_file" ]; then
|
||||
echo "ERROR: Password file not found at $password_file" | tee -a "$LOG_FILE"
|
||||
if [ ! -f "$PASSWORD_FILE" ]; then
|
||||
log "ERROR: Password file not found at $PASSWORD_FILE."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export the password from the file to ensure Restic uses it automatically
|
||||
export RESTIC_PASSWORD=$(cat "$PASSWORD_FILE")
|
||||
|
||||
# Verify repository access
|
||||
if ! restic snapshots > /dev/null 2>&1; then
|
||||
echo "ERROR: Unable to access the restic repository. Aborting backup." | tee -a "$LOG_FILE"
|
||||
if ! restic -r "$BACKUP_PATH" snapshots > /dev/null 2>&1; then
|
||||
log "ERROR: Unable to access the Restic repository. Aborting backup."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Acquire a global lock to serialize Restic operations
|
||||
log "Acquiring global lock for Restic operations..."
|
||||
exec 9>"$LOCK_FILE"
|
||||
if ! flock -n 9; then
|
||||
log "Another Restic operation is running. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
log "Global lock acquired."
|
||||
|
||||
# Check and remove stale locks
|
||||
log "Checking for stale locks in the repository..."
|
||||
if restic -r "$BACKUP_PATH" list locks | grep -q "lock"; then
|
||||
log "Stale locks detected. Unlocking the repository..."
|
||||
restic -r "$BACKUP_PATH" unlock
|
||||
log "Repository unlocked successfully."
|
||||
else
|
||||
log "No stale locks found."
|
||||
fi
|
||||
|
||||
# Check and fix permissions on /mnt/backups
|
||||
if [ ! -w "$BACKUP_PATH/locks" ]; then
|
||||
log "Fixing permissions on $BACKUP_PATH/locks"
|
||||
sudo chown -R litespeed:litespeed "$BACKUP_PATH/locks"
|
||||
sudo chmod -R u+rw "$BACKUP_PATH/locks"
|
||||
fi
|
||||
|
||||
# Perform the backup
|
||||
for path in "${includePaths[@]}"; do
|
||||
if restic backup "$path" \
|
||||
for path in "${INCLUDE_PATHS[@]}"; do
|
||||
log "Starting backup for $path..."
|
||||
if restic -r "$BACKUP_PATH" backup "$path" \
|
||||
--tag media_themes \
|
||||
--tag "$CUSTOM_TAG" \
|
||||
--force \
|
||||
--option b2.connections=4; then
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] Backup completed successfully for $path with tags: media_themes, $CUSTOM_TAG." | tee -a "$LOG_FILE"
|
||||
log "Backup completed successfully for $path with tags: media_themes, $CUSTOM_TAG."
|
||||
else
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: Backup failed for $path." | tee -a "$LOG_FILE"
|
||||
log "ERROR: Backup failed for $path."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Release global lock
|
||||
exec 9>&-
|
||||
|
||||
# End logging
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] Media Backup process finished." | tee -a "$LOG_FILE"
|
||||
log "Media Backup process finished successfully."
|
||||
|
|
|
|||
Loading…
Reference in New Issue