mb-admin/scripts/ssl-manager/xmlchecker.sh

#!/bin/bash
# ==============================================================================
# Script Name: xmlchecker.sh
# Description: Validates and attempts to fix structural errors in LiteSpeed's 
#              httpd_config.xml file. Includes robust fallback mechanisms, 
#              semantic validation, and service restart verification.
# Version: 1.8.0 (Enterprise-Grade Optimized)
# Author: Anthony Garces (tony@mightybox.io)
# Date: 2025-03-26
# Exit Codes:
#   0: Success (original valid or minor fixes applied, or default/backup applied successfully)
#   1: General Error / Fallback to Backup completed (review recommended) / Restart failed
#   2: Success, but MINIMAL fallback config applied (CRITICAL REVIEW NEEDED)
#   3: Success, but RECOVERED config with MAJOR changes applied (CRITICAL REVIEW NEEDED)
# ==============================================================================
set -euo pipefail

# === Configuration ===
CONF_FILE="/var/www/conf/httpd_config.xml"
DEFAULT_CONF="/var/www/conf/httpd_config.default.xml" # Official default file path
SERVER_ROOT="/var/www" # Server root directory
LOG_DIR="/var/log/mb-ssl" # Log directory
MAJOR_CHANGE_THRESHOLD=20 # Lines changed to trigger "major recovery" warning

# Log files
SCRIPT_LOG="${LOG_DIR}/xmlchecker.log"
ERROR_LOG="${LOG_DIR}/xmlchecker-error.log"
DEBUG_LOG="${LOG_DIR}/xmlchecker-debug.log"
BACKUP_FILE="${LOG_DIR}/httpd_config_$(date +%Y%m%d_%H%M%S).bak" # Timestamped backup

# SSL Remover script path
SSL_REMOVER="/home/litespeed/mbmanager/ssl-manager/ssl_remover.sh"

# - Internal Variables -
TMP_FILE="/tmp/test_config_$(date +%s).xml" # Temporary copy of original/cleaned file
RECOVERY_TMP_FILE="/tmp/recovery_test_config_$(date +%s).xml" # Holds output of xmllint --recover
MINIMAL_TMP="/tmp/minimal_config_$(date +%s).xml" # Temporary minimal config file
VERBOSE=0
INITIAL_FILE_VALID=0 # Flag: 1 if CONF_FILE was valid before fixes
RECOVERY_PRODUCED_VALID_XML=0 # Flag: 1 if xmllint --recover output was valid XML
RECOVERY_WAS_MAJOR=0 # Flag: 1 if recovery resulted in significant changes
APPLIED_CONFIG_SOURCE="none" # Tracks what was finally applied

# === Cleanup Trap ===
trap 'log "Script interrupted. Cleaning up temporary files..."; sudo rm -f "$TMP_FILE" "$RECOVERY_TMP_FILE" "$MINIMAL_TMP"' EXIT

# === Functions ===
setup_logging() {
    # Create log directory if it doesn't exist
    sudo mkdir -p "$LOG_DIR" || { echo "❌ ERROR: Cannot create log directory '$LOG_DIR'. Check permissions."; exit 1; }
    
    # Set proper permissions
    sudo chown -R "$(whoami)":"$(id -gn)" "$LOG_DIR"
    sudo chmod 755 "$LOG_DIR"
    
    # Create log files with proper permissions
    touch "$SCRIPT_LOG" "$ERROR_LOG" "$DEBUG_LOG"
    chmod 644 "$SCRIPT_LOG" "$ERROR_LOG" "$DEBUG_LOG"
    
    # Add log rotation if log files are too large (>10MB)
    for log_file in "$SCRIPT_LOG" "$ERROR_LOG" "$DEBUG_LOG"; do
        if [ -f "$log_file" ] && [ "$(stat -f%z "$log_file" 2>/dev/null || stat -c%s "$log_file")" -gt 10485760 ]; then
            mv "$log_file" "${log_file}.$(date +%Y%m%d)"
            touch "$log_file"
            chmod 644 "$log_file"
            gzip "${log_file}.$(date +%Y%m%d)"
        fi
    done
}

log() {
    local level="INFO"
    local message="$1"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    
    # Log to main log file
    echo "[$timestamp] [$level] $message" | sudo tee -a "$SCRIPT_LOG"
    
    # Log errors to error log file
    if [[ "$message" == *"ERROR"* ]] || [[ "$message" == *"❌"* ]]; then
        echo "[$timestamp] [$level] $message" | sudo tee -a "$ERROR_LOG"
    fi
}

log_verbose() {
    if [[ "$VERBOSE" -eq 1 ]]; then
        local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
        echo "[$timestamp] [DEBUG] $1" | sudo tee -a "$DEBUG_LOG"
    fi
}

log_error() {
    local message="$1"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo "[$timestamp] [ERROR] $message" | sudo tee -a "$ERROR_LOG" "$SCRIPT_LOG"
}

log_success() {
    local message="$1"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo "[$timestamp] [SUCCESS] $message" | sudo tee -a "$SCRIPT_LOG"
}

check_command() {
    local cmd="$1"
    local pkg="$2"
    if ! command -v "$cmd" &>/dev/null; then
        log "⚠ Required command '$cmd' not found. Attempting to install package '$pkg'..."
        
        # Try dnf first (AlmaLinux)
        if command -v dnf &>/dev/null; then
            if sudo dnf install -y "$pkg"; then
                log "✔ Successfully installed '$pkg' using dnf."
                return 0
            fi
        fi
        
        # Fallback to yum (CentOS)
        if command -v yum &>/dev/null; then
            if sudo yum install -y "$pkg"; then
                log "✔ Successfully installed '$pkg' using yum."
                return 0
            fi
        fi
        
        log "❌ ERROR: Failed to install '$pkg' using either dnf or yum. Exiting."
        exit 1
    fi
}

validate_listener_certificates() {
    local file_to_check="$1"
    log "Validating listener certificates in '$file_to_check'..."
    local invalid_listeners=()

    # Get all HTTPS listeners
    local listeners
    listeners=$(sudo xmlstarlet sel -Q -t -v "//listenerList/listener[contains(name, 'HTTPS-')]/name" "$file_to_check" 2>/dev/null || echo "")
    
    for listener in $listeners; do
        local domain="${listener#HTTPS-}"
        local key_file=$(sudo xmlstarlet sel -Q -t -v "//listenerList/listener[name='$listener']/keyFile" "$file_to_check" 2>/dev/null || echo "")
        local cert_file=$(sudo xmlstarlet sel -Q -t -v "//listenerList/listener[name='$listener']/certFile" "$file_to_check" 2>/dev/null || echo "")

        # Check if certificate files exist
        if [[ ! -f "$key_file" ]] || [[ ! -f "$cert_file" ]]; then
            log "⚠ Found invalid listener '$listener' with missing certificate files"
            log_verbose "Key file missing: $key_file"
            log_verbose "Cert file missing: $cert_file"
            invalid_listeners+=("$domain")
        fi

        # Check if certificate is still valid
        if [[ -f "$cert_file" ]]; then
            local cert_expiry
            cert_expiry=$(openssl x509 -enddate -noout -in "$cert_file" | cut -d= -f2)
            local expiry_date
            expiry_date=$(date -d "$cert_expiry" +%s)
            local current_date
            current_date=$(date +%s)

            if [[ $expiry_date -lt $current_date ]]; then
                log "⚠ Found expired certificate for listener '$listener'"
                log_verbose "Certificate expired on: $cert_expiry"
                invalid_listeners+=("$domain")
            fi
        fi
    done

    # If we found invalid listeners, clean them up
    if [[ ${#invalid_listeners[@]} -gt 0 ]]; then
        log "Found ${#invalid_listeners[@]} invalid listener(s). Cleaning up..."
        for domain in "${invalid_listeners[@]}"; do
            log "Cleaning up invalid listener for domain: $domain"
            if [[ -f "$SSL_REMOVER" ]]; then
                log "Running SSL remover for domain: $domain"
                sudo bash "$SSL_REMOVER" --domains="$domain" --verbose || {
                    log_error "Failed to clean up invalid listener for domain: $domain"
                }
            else
                log_error "SSL remover script not found at: $SSL_REMOVER"
                # Fallback to manual cleanup
                sudo xmlstarlet ed -L \
                    -d "//listenerList/listener[name='HTTPS-$domain']" \
                    "$file_to_check" || {
                        log_error "Failed to remove invalid listener for domain: $domain"
                    }
            fi
        done
    fi

    return ${#invalid_listeners[@]}
}

perform_semantic_checks() {
    local file_to_check="$1"
    log_verbose "Performing semantic checks on '$file_to_check'..."
    local semantic_errors=0

    # Check for critical elements
    if ! sudo xmlstarlet sel -t -c "//virtualHostList" "$file_to_check" &>/dev/null; then
        log_error "Missing <virtualHostList> in '$file_to_check'"
        semantic_errors=$((semantic_errors + 1))
    fi
    if ! sudo xmlstarlet sel -t -c "//listenerList" "$file_to_check" &>/dev/null; then
        log_error "Missing <listenerList> in '$file_to_check'"
        semantic_errors=$((semantic_errors + 1))
    fi

    # Validate listener certificates
    if ! validate_listener_certificates "$file_to_check"; then
        semantic_errors=$((semantic_errors + 1))
    fi

    # Return result
    if [[ "$semantic_errors" -eq 0 ]]; then
        log_success "Semantic checks passed for '$file_to_check'"
        return 0
    else
        log_error "Found $semantic_errors semantic issue(s) in '$file_to_check'"
        return 1
    fi
}

clean_invalid_entries() {
    local file_to_clean="$1"
    log "Attempting to clean invalid entries from '$file_to_clean'..."

    # Remove empty or redundant nodes
    sudo xmlstarlet ed -L -d '//virtualHost[not(node()[not(self::comment() or (self::text() and normalize-space()=""))])]' "$file_to_clean" || log "Warning: Failed to clean empty virtualHost nodes."
    sudo xmlstarlet ed -L -d '//listener[not(node()[not(self::comment() or (self::text() and normalize-space()=""))])]' "$file_to_clean" || log "Warning: Failed to clean empty listener nodes."
    sudo xmlstarlet ed -L -d '//virtualHostList[not(node()[not(self::comment() or (self::text() and normalize-space()=""))])]' "$file_to_clean" || log "Warning: Failed to clean empty virtualHostList."
    sudo xmlstarlet ed -L -d '//listenerList[not(node()[not(self::comment() or (self::text() and normalize-space()=""))])]' "$file_to_clean" || log "Warning: Failed to clean empty listenerList."

    log "Cleanup attempts completed for '$file_to_clean'."
}

restart_litespeed() {
    log "Attempting to restart LiteSpeed server..."
    if sudo systemctl restart lsws; then
        log "Info: Restart command issued for LiteSpeed (lsws). Waiting 3 seconds to verify status..."
        sleep 3
        if sudo systemctl is-active --quiet lsws; then
            log "✔ LiteSpeed (lsws) is active after restart."
            return 0
        else
            log "❌ FATAL: LiteSpeed (lsws) failed to become active after restart command."
            log "         Check service status: 'sudo systemctl status lsws'"
            log "         Check LiteSpeed error log: '${SERVER_ROOT:-/var/www}/logs/error.log'"
            log "         Consider manually reverting to backup: sudo cp '$BACKUP_FILE' '$CONF_FILE' && sudo systemctl restart lsws"
            return 1
        fi
    else
        log "❌ FATAL: Failed to execute restart command for LiteSpeed (lsws)."
        return 1
    fi
}

apply_and_restart() {
    local source_file="$1"
    local source_desc="$2" # e.g., "original", "recovered", "default", "backup", "minimal"

    APPLIED_CONFIG_SOURCE="$source_desc" # Update global tracker

    log "Applying changes from $source_desc file ('$source_file') to '$CONF_FILE'..."

    # Check if source and destination are the same file
    if [[ "$source_file" == "$CONF_FILE" ]]; then
        log "Info: Source file ('$source_file') and destination file ('$CONF_FILE') are the same. Skipping copy."
    else
        log "Preview of changes (diff with backup '$BACKUP_FILE'):"
        if ! sudo diff -u "$BACKUP_FILE" "$source_file" | sudo tee -a "$SCRIPT_LOG"; then
            log "Info: No differences found or diff command failed."
        fi

        # Apply the configuration using sudo cp
        if sudo cp -a "$source_file" "$CONF_FILE"; then
            log "✔ Configuration updated successfully from $source_desc file."
        else
            log "❌ FATAL: Failed to copy $source_desc file '$source_file' to '$CONF_FILE'. Configuration NOT updated."
            return 1 # Signal failure
        fi
    fi

    # Restart LiteSpeed using the hardcoded command
    log "Attempting to restart LiteSpeed server..."
    if sudo systemctl restart lsws; then
        log "Info: Restart command issued for LiteSpeed (lsws). Waiting 3 seconds to verify status..."
        sleep 3
        if sudo systemctl is-active --quiet lsws; then
            log "✔ LiteSpeed (lsws) is active after restart."
            return 0 # Success
        else
            log "❌ FATAL: LiteSpeed (lsws) failed to become active after restart command."
            log "         Check service status: 'sudo systemctl status lsws'"
            log "         Check LiteSpeed error log: '${SERVER_ROOT:-/var/www}/logs/error.log'"
            log "         Consider manually reverting to backup: sudo cp '$BACKUP_FILE' '$CONF_FILE' && sudo systemctl restart lsws"
            return 1 # Failure
        fi
    else
        log "❌ FATAL: Failed to execute restart command for LiteSpeed (lsws)."
        return 1 # Failure
    fi
}

# === Main Script Logic ===
main() {
    # Setup logging first
    setup_logging
    log "Starting XML config check/fix V1.8.0 for $CONF_FILE"

    # Argument Parsing
    if [[ "${1:-}" == "--verbose" ]]; then
        VERBOSE=1
        log "Verbose mode enabled"
    fi

    # Dependency Checks
    log_verbose "Checking dependencies..."
    check_command "xmllint" "libxml2-utils"
    check_command "xmlstarlet" "xmlstarlet"

    # Backup Original File
    log "Backing up '$CONF_FILE' to '$BACKUP_FILE'..."
    sudo cp -a "$CONF_FILE" "$BACKUP_FILE"

    # Initial Validation
    log_verbose "Running initial xmllint validation on '$CONF_FILE'..."
    if sudo xmllint --noout "$CONF_FILE" 2>/dev/null; then
        log_success "Initial Check: '$CONF_FILE' is valid XML"
        INITIAL_FILE_VALID=1
    else
        log_error "Initial Check: '$CONF_FILE' is invalid XML. Will attempt recovery/cleanup"
    fi

    # Recovery Attempt
    if [[ "$INITIAL_FILE_VALID" -eq 0 ]]; then
        log "Attempting automatic recovery with 'xmllint --recover'..."
        sudo rm -f "$RECOVERY_TMP_FILE"
        if xmllint --recover "$CONF_FILE" 2>/dev/null > "$RECOVERY_TMP_FILE"; then
            if sudo xmllint --noout "$RECOVERY_TMP_FILE" 2>/dev/null; then
                log_success "'xmllint --recover' produced structurally valid XML output"
                RECOVERY_PRODUCED_VALID_XML=1
            else
                log_error "'xmllint --recover' ran but the output is STILL invalid XML. Discarding recovery"
                sudo rm -f "$RECOVERY_TMP_FILE"
            fi
        else
            log_error "'xmllint --recover' command failed or produced errors. Discarding recovery"
            sudo rm -f "$RECOVERY_TMP_FILE"
        fi
    fi

    # Final Validation and Decision
    FINAL_CONFIG_SOURCE_FILE=""
    FINAL_CONFIG_SOURCE_DESC=""
    FINAL_EXIT_CODE=0
    log_verbose "Running final validation and decision logic..."

    # Check 1: Is the original (potentially cleaned) temp file valid now?
    if [[ "$INITIAL_FILE_VALID" -eq 1 ]]; then
        FINAL_CONFIG_SOURCE_FILE="$CONF_FILE"
        FINAL_CONFIG_SOURCE_DESC="original"
    elif [[ "$RECOVERY_PRODUCED_VALID_XML" -eq 1 ]]; then
        FINAL_CONFIG_SOURCE_FILE="$RECOVERY_TMP_FILE"
        FINAL_CONFIG_SOURCE_DESC="recovered"
    else
        log_error "Neither original nor recovered file is valid. Falling back to default or minimal config"
        FINAL_EXIT_CODE=2
    fi

    # Apply and Restart
    if [[ -n "$FINAL_CONFIG_SOURCE_FILE" ]]; then
        if apply_and_restart "$FINAL_CONFIG_SOURCE_FILE" "$FINAL_CONFIG_SOURCE_DESC"; then
            log_success "Script completed successfully. Applied: $FINAL_CONFIG_SOURCE_DESC"
            exit "$FINAL_EXIT_CODE"
        else
            log_error "Failed to apply and restart using $FINAL_CONFIG_SOURCE_DESC config"
            exit 1
        fi
    else
        log_error "No valid configuration file found. Exiting"
        exit 1
    fi
}

# === Entry Point ===
main "$@"

# === Fallback Logic ===
log "Attempting fallback sequence..."
FALLBACK_APPLIED="" # Track which fallback succeeded

# Priority 1: Official Default Config (with include checks)
if [[ -z "$FALLBACK_APPLIED" && -f "$DEFAULT_CONF" ]]; then
    log_verbose "Fallback Priority 1: Checking official default config '$DEFAULT_CONF'..."
    if sudo xmllint --noout "$DEFAULT_CONF" 2>/dev/null; then
        # Validate semantic correctness of the default config
        if perform_semantic_checks "$DEFAULT_CONF"; then
            log "Info: Default config is valid. Applying default configuration."

            # Log the exact content of the default config for debugging
            log_verbose "Exact content of default config ('$DEFAULT_CONF'):"
            sudo cat "$DEFAULT_CONF" | sudo tee -a "$SCRIPT_LOG" >/dev/null

            # Apply the default configuration
            if apply_and_restart "$DEFAULT_CONF" "default"; then
                log "✅ Script completed successfully. Applied: default configuration."

                # Verify the copied file matches the default config exactly
                if ! sudo diff -q "$DEFAULT_CONF" "$CONF_FILE" &>/dev/null; then
                    log "❌ FATAL: Copied file does NOT match the default configuration exactly!"
                    log "         Differences detected between '$DEFAULT_CONF' and '$CONF_FILE'."
                    exit 1
                else
                    log "✔ Verified: Copied file matches the default configuration exactly."
                fi

                FALLBACK_APPLIED="default"
                exit 0 # Exit code 0 for success with default config
            else
                log "❌ FATAL: Failed to apply/restart using default configuration!"
            fi
        else
            log "⚠ Semantic Check: Default config failed semantic validation. Skipping fallback to default."
        fi
    else
        log "⚠ Structural Check: Default config failed XML validation. Skipping fallback to default."
    fi
fi

# Priority 2: Minimal Fallback Configuration
if [[ -z "$FALLBACK_APPLIED" ]]; then
    log "Fallback Priority 2: Generating minimal fallback configuration..."
    cat <<EOF | sudo tee "$MINIMAL_TMP" >/dev/null
<?xml version="1.0" encoding="UTF-8"?>
<httpServerConfig>
    <serverName>LiteSpeed</serverName>
    <listenerList>
        <listener>
            <name>HTTP</name>
            <address>*:80</address>
        </listener>
    </listenerList>
    <virtualHostList>
        <virtualHost>
            <name>default</name>
            <vhRoot>/var/www/webroot/</vhRoot>
            <configFile>\$SERVER_ROOT/conf/vhconf.xml</configFile>
            <allowSymbolLink>1</allowSymbolLink>
            <enableScript>1</enableScript>
            <restrained>1</restrained>
            <setUIDMode>0</setUIDMode>
            <chrootMode>0</chrootMode>
        </virtualHost>
    </virtualHostList>
    <extProcessorList>
        <extProcessor>
            <type>lsapi</type>
            <name>lsphp</name>
            <address>uds://tmp/lshttpd/lsphp.sock</address>
            <maxConns>35</maxConns>
            <env>PHP_LSAPI_MAX_REQUESTS=500</env>
            <initTimeout>60</initTimeout>
            <retryTimeout>0</retryTimeout>
            <persistConn>1</persistConn>
        </extProcessor>
    </extProcessorList>
</httpServerConfig>
EOF

    # Validate minimal config
    if sudo xmllint --noout "$MINIMAL_TMP" 2>/dev/null; then
        if perform_semantic_checks "$MINIMAL_TMP"; then
            log "Info: Minimal fallback config generated successfully and passed validation."
            if apply_and_restart "$MINIMAL_TMP" "minimal"; then
                log "🚨 CRITICAL WARNING: Server running on MINIMAL config. Manual reconfiguration required! 🚨"
                log "✅ Script completed. Applied: minimal configuration."
                FALLBACK_APPLIED="minimal"
                exit 2 # Exit code 2 for minimal config
            else
                log "❌ FATAL: Failed to apply/restart even the minimal config!"
            fi
        else
            log "❌ FATAL: Generated minimal config failed semantic validation! This is a script bug."
        fi
    else
        log "❌ FATAL: Generated minimal config is invalid XML! This is a script bug."
    fi
fi

# If we reach here, all fallbacks failed
log "❌ FATAL: All fallback mechanisms failed. Server configuration remains unchanged."
exit 1