Haiyang Qiu

import os
import sys
from collections import defaultdict
import tempfile
from magika import Magika
import py7zr
import pyzipper
import shutil

"""
This script is used to sort files by their type. 
It reads all the files in the provided directory and splits them into different 
directories based on their type. Mainly, it classifies the files into the following types:
- PE Binaries
- Mach-O Binaries
- ELF Binaries
- APK 
- Other Files

It outputs the count of files for each file extension and file type.

Command line example:
nohup python3 sort_files_by_type.py /mnt/samples/consilience/malware/vx-bazaar-expanded/Bazaar.2020.12 \
    /mnt/samples/consilience/malware/_sorted_samples/vx-bazaar/2020/ > \
    /mnt/samples/consilience/malware/ops-logs/20241014-SORT_FILES-bazaar_2020.12.output.log 2>&1 &
"""


def process_zip_file(filepath):
    print(f"[INFO] - Processing zip file: {filepath}")
    with tempfile.TemporaryDirectory() as temp_dir:
        try:
            with pyzipper.AESZipFile(filepath) as zf:
                zf.pwd = b"infected"
                filename = zf.namelist()[0]
                zf.extractall(temp_dir)
                extracted_zip_path = os.path.join(temp_dir, filename)
                return process_binary_file(extracted_zip_path)
        except Exception as e:
            print(f"[ERR] - Error processing zip file {filepath}: {str(e)}")
            return None


def process_7z_file(filepath):
    print(f"[INFO] - Processing 7z file: {filepath}")
    with tempfile.TemporaryDirectory() as temp_dir:
        try:
            with py7zr.SevenZipFile(filepath, mode="r", password="infected") as z:
                z.extractall(path=temp_dir)
                for root, _, files in os.walk(temp_dir):
                    for file in files:
                        extracted_file_path = os.path.join(root, file)
                        return process_binary_file(extracted_file_path)
        except Exception as e:
            print(f"[ERR] - Error processing 7z file {filepath}: {str(e)}")
            return None


def process_binary_file(filepath):
    print(f"[INFO] - Processing binary file: {filepath}")
    try:
        with open(filepath, "rb") as f:
            data = f.read(512 * 1024)  # 512KB
        return Magika().identify_bytes(data).output.ct_label
    except Exception as e:
        print(f"[ERR] - Error processing binary file {filepath}: {str(e)}")
        return "ERROR"


def count_file_extensions(path, dest_dir):
    # A defaultdict to store file extension counts
    extension_count = defaultdict(int)
    filetype_count = defaultdict(int)
    tot_files = 0

    # Walk through the directory
    for root, dirs, files in os.walk(path):
        for file in files:
            if file.startswith("."):
                continue

            tot_files += 1
            # Split the file extension from the file name
            _, ext = os.path.splitext(file)
            ext = ext.lower()

            # Count the extensions
            extension_count[ext] += 1
            filepath = os.path.join(root, file)
            filetype = ""

            if ext == ".7z":
                filetype = process_7z_file(filepath)
            elif ext == ".zip":
                filetype = process_zip_file(filepath)
            else:
                filetype = process_binary_file(filepath)

            filetype_count[filetype] += 1

            if filetype == "pebin":
                dest = os.path.join(dest_dir, "pebin")
                os.makedirs(dest, exist_ok=True)
                shutil.copy2(filepath, os.path.join(dest, file))
            elif filetype == "macho":
                dest = os.path.join(dest_dir, "macho")
                os.makedirs(dest, exist_ok=True)
                shutil.copy2(filepath, os.path.join(dest, file))
            elif filetype == "elf":
                dest = os.path.join(dest_dir, "elf")
                os.makedirs(dest, exist_ok=True)
                shutil.copy2(filepath, os.path.join(dest, file))
            elif filetype == "apk":
                dest = os.path.join(dest_dir, "apk")
                os.makedirs(dest, exist_ok=True)
                shutil.copy2(filepath, os.path.join(dest, file))
            else:
                dest = os.path.join(dest_dir, "other")
                os.makedirs(dest, exist_ok=True)
                shutil.copy2(filepath, os.path.join(dest, file))
            print(f"[STATUS] - Progres: {tot_files}/{len(files)} files processed")

    # Print the results
    # print("- File Extension Statistics:")
    # for ext, count in extension_count.items():
    #     if ext:  # To exclude files with no extension
    #         print(f"\t{ext}: {count} files")

    # if "" in extension_count:
    #     print(f'\t{extension_count[""]} files with no extension')

    print(f"- Total Number of Files: {tot_files}")
    print("\n- File Type Statistics:")
    for filetype, count in sorted(filetype_count.items()):
        print(f"\t{filetype:<15}{count:>8}")


if __name__ == "__main__":
    # Check if the path was passed as a command-line argument
    if len(sys.argv) != 3:
        print("Usage: python3 script.py <input_path> <dest_dir>")
        sys.exit(1)

    # Get the path from the command-line arguments
    input_path = sys.argv[1]
    dest_dir = sys.argv[2]

    # Check if the provided path is valid
    if not os.path.isdir(input_path):
        print(f"The path '{input_path}' is not a valid directory.")
        sys.exit(1)

    # Call the function to count file extensions
    count_file_extensions(input_path, dest_dir)