generate.sh impro

[fabriziosalmi]

#!/bin/bash

# Exit on error and unset variables
set -euo pipefail

# Configuration
LOGFILE="setup_script.log"
BLACKLIST_URLS_FILE="blacklists.fqdn.urls"
MAX_CONCURRENT_DOWNLOADS=5

# Colors for better user feedback
GREEN="\e[32m"
RED="\e[31m"
YELLOW="\e[33m"
RESET="\e[0m"

# Function to log messages with timestamps
log() {
    local message="$1"
    echo -e "$(date '+%Y-%m-%d %H:%M:%S') - ${message}${RESET}" | tee -a "$LOGFILE"
}

# Function to log success messages
log_success() {
    log "${GREEN}✅ $1"
}

# Function to log error messages and exit
log_error() {
    log "${RED}❌ $1"
    exit 1
}

# Function to log warnings
log_warning() {
    log "${YELLOW}⚠️  $1"
}

# Detect package manager and configure commands for package operations
detect_package_manager() {
    if command -v apt-get &>/dev/null; then
        PACKAGE_MANAGER="apt-get"
        UPDATE_CMD="sudo apt-get update"
        INSTALL_CMD="sudo apt-get install -y"
    elif command -v apk &>/dev/null; then
        PACKAGE_MANAGER="apk"
        UPDATE_CMD="sudo apk update"
        INSTALL_CMD="sudo apk add --no-cache"
    elif command -v yum &>/dev/null; then
        PACKAGE_MANAGER="yum"
        UPDATE_CMD="sudo yum check-update"
        INSTALL_CMD="sudo yum install -y"
    elif command -v brew &>/dev/null; then
        PACKAGE_MANAGER="brew"
        UPDATE_CMD="brew update"
        INSTALL_CMD="brew install"
    else
        log_error "Unsupported package manager. Exiting."
    fi
    log_success "Detected package manager: $PACKAGE_MANAGER"
}

# Update and install prerequisites
update_and_install() {
    log "Updating system and installing Python 3..."
    if ! $UPDATE_CMD | tee -a "$LOGFILE"; then
        log_error "Failed to update system."
    fi
    if ! $INSTALL_CMD python3 | tee -a "$LOGFILE"; then
        log_error "Failed to install Python 3."
    fi
    if [ "$PACKAGE_MANAGER" == "apt-get" ]; then
        sudo ln -sf /usr/bin/python3 /usr/bin/python
    fi
    if ! python3 -m ensurepip --upgrade | tee -a "$LOGFILE"; then
        log_error "Failed to upgrade pip."
    fi
    if ! pip3 install --no-cache-dir --upgrade pip setuptools tldextract tqdm | tee -a "$LOGFILE"; then
        log_error "Failed to install Python packages."
    fi
    log_success "System updated and Python 3 installed."
}

# Install additional required packages
install_additional_packages() {
    for package in pv ncftp; do
        log "Installing package: $package..."
        if ! $INSTALL_CMD "$package" | tee -a "$LOGFILE"; then
            log_error "Failed to install '$package' using $PACKAGE_MANAGER."
        fi
    done
    log_success "Additional packages installed."
}

# Function to download a URL and save to a randomly named file
download_url() {
    local url="$1"
    local random_filename
    random_filename=$(mktemp --suffix=".fqdn.list")
    log "Downloading blacklist: $url -> $random_filename"
    if ! wget -q --timeout=30 --tries=3 --progress=bar:force -O "$random_filename" "$url"; then
        log_warning "Failed to download: $url"
        rm -f "$random_filename"
        return 1
    fi
    echo "$random_filename"
}

# Download all URLs from the list and handle files
manage_downloads() {
    if [ ! -f "$BLACKLIST_URLS_FILE" ]; then
        log_error "File $BLACKLIST_URLS_FILE not found."
    fi
    log "Starting downloads with a maximum of $MAX_CONCURRENT_DOWNLOADS concurrent downloads..."
    local download_count=0
    while IFS= read -r url; do
        if (( download_count >= MAX_CONCURRENT_DOWNLOADS )); then
            wait -n
            (( download_count-- ))
        fi
        download_url "$url" &
        (( download_count++ ))
    done < "$BLACKLIST_URLS_FILE"
    wait
    log_success "All downloads completed."
}

# Sanitize and whitelist downloaded blacklists
sanitize_and_whitelist() {
    log "Sanitizing blacklists..."
    if [ -f sanitize.py ]; then
        if ! python3 sanitize.py | tee -a "$LOGFILE"; then
            log_error "Failed to sanitize blacklists."
        fi
    else
        log_warning "sanitize.py not found. Skipping sanitation."
    fi
    log "Removing whitelisted domains..."
    if [ -f whitelist.py ]; then
        if ! python3 whitelist.py | tee -a "$LOGFILE"; then
            log_error "Failed to filter whitelisted domains."
        fi
    else
        log_warning "whitelist.py not found. Skipping whitelist filtering."
    fi
    log_success "Blacklists sanitized and whitelisted."
}

# Main routine
main() {
    log "Starting setup script 🛠️"
    detect_package_manager
    update_and_install
    install_additional_packages
    manage_downloads
    sanitize_and_whitelist
    local total_lines
    total_lines=$(wc -l < all.fqdn.blacklist 2>/dev/null || echo 0)
    log_success "Total domains: $total_lines 🌍"
    log "Setup script completed successfully."
}

# Run the main function
main