J. Doyne Farmer

import os
import subprocess
import multiprocessing
import logging
from concurrent.futures import ThreadPoolExecutor
import signal

# Variables (replace these with actual values)
src_root_folder_path = "/mnt/samples/consilience/malware/vx-bazaar-expanded/"
dst_root_folder_path = "/mnt/samples/consilience/malware/_sorted_samples/vx-bazaar/"
log_folder_path = "/mnt/samples/consilience/malware/ops-logs/"
folder_names = ["Bazaar.2022.01", "Bazaar.2022.02", "Bazaar.2022.03", "Bazaar.2022.04"]  # List of folder names
script_name = "sort_files_by_type.py"
dry_run = True  # Set to True for testing without execution


# Ensure the log folder exists
os.makedirs(log_folder_path, exist_ok=True)

# Configure logging
main_log_name = "20241229-main_script.log"
main_log_file = os.path.join(log_folder_path, main_log_name)
if not os.path.exists(main_log_file):
    open(main_log_file, 'a').close()  # Create the log file if it doesn't exist
# main_log_file = os.path.join(log_folder_path, main_log_name)
# try:
#     with open(main_log_file, 'a') as f:
#         print(f"Successfully created/opened log file at: {main_log_file}")
#     # Verify the file exists
#     if os.path.exists(main_log_file):
#         print(f"Confirmed log file exists at: {main_log_file}")
#         print(f"File size: {os.path.getsize(main_log_file)} bytes")
# except Exception as e:
#     print(f"Error with log file: {e}")

# Configure logging
logging.basicConfig(
    filename=main_log_file,
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    force=True,
    filemode='a'
)

logging.info("Logging initialization test message")

# Add logging handler to flush immediately
class FlushableFileHandler(logging.FileHandler):
    def emit(self, record):
        super().emit(record)
        self.flush()

for handler in logging.getLogger().handlers:
    if isinstance(handler, logging.FileHandler):
        handler.flush()

# Global flag for graceful termination
terminate_flag = False

def signal_handler(signum, frame):
    global terminate_flag
    terminate_flag = True
    logging.warning("Termination signal received. Attempting to shut down gracefully.")

# Register signal handler
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)

# Define the worker function
def process_folder(folder):
    global terminate_flag
    if terminate_flag:
        logging.warning(f"Skipping folder {folder} due to termination signal.")
        return

    try:
        # Extract the word and date fields from the folder name
        word, year, month = folder.split('.')

        # Construct source, destination, and log paths
        src_folder = os.path.join(src_root_folder_path, folder)
        dst_folder = os.path.join(dst_root_folder_path, folder)
        log_file = os.path.join(
            log_folder_path, f"20241229-SORT_FILES-bazaar_{year}.{month}.output.log"
        )

        if not os.path.exists(log_file):
            open(log_file, 'a').close()  # Create the log file if it doesn't exist

        # Build the command
        command = ["python3", script_name, src_folder, dst_folder]

        if dry_run:
            logging.info(f"Dry run: Would execute: {command}, log: {log_file}")
            print(f"Dry run: Would execute: {command}, log: {log_file}")
        else:
            logging.info(f"Starting task for folder: {folder}")
            with open(log_file, "w") as log:
                subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=True)
            logging.info(f"Task completed for folder: {folder}")

    except subprocess.CalledProcessError as e:
        logging.error(f"Subprocess failed for folder {folder} with return code {e.returncode}: {e}")
        print(f"Subprocess failed for folder {folder}: {e}")
    except Exception as e:
        logging.error(f"Error while processing folder {folder}: {e}")
        print(f"Error while processing folder {folder}: {e}")

# Determine the number of processors
num_processors = multiprocessing.cpu_count()

try:
    # Process folders using ThreadPoolExecutor
    with ThreadPoolExecutor(max_workers=num_processors-1) as executor:
        executor.map(process_folder, folder_names)

except Exception as e:
    logging.critical(f"Critical error in the main execution: {e}")
    print(f"Critical error: {e}")

logging.info("All tasks completed.")
print("All tasks completed.")