Wai Keung Wong

import os
import time
import json
import hashlib
import re
import threading
from enum import Enum
from typing import Dict, List, Any, Optional, Set, Tuple
from datetime import datetime, timezone
import textwrap
import logging

from bninja.utils.license import set_license

# Binary Ninja imports (conditional)
try:
    import binaryninja
    from binaryninja import BinaryView, BasicBlock, Function, core_ui_enabled
    from binaryninja import MediumLevelILInstruction, LowLevelILInstruction
    from binaryninja import mainthread
    from binaryninja.enums import (
        InstructionTextTokenType,
        FunctionAnalysisSkipOverride,
        BranchType,
    )
    from binaryninja.types import Symbol

    BINARYNINJA_AVAILABLE = True
except ImportError:
    # Binary Ninja not available - define dummy classes/constants
    BINARYNINJA_AVAILABLE = False
    binaryninja = None
    BinaryView = None
    BasicBlock = None
    Function = None
    core_ui_enabled = None
    MediumLevelILInstruction = None
    LowLevelILInstruction = None
    mainthread = None
    InstructionTextTokenType = None
    FunctionAnalysisSkipOverride = None
    BranchType = None
    Symbol = None


class InstructionType(Enum):
    # Data Movement
    GENERAL_DATA_MOVEMENT = "GENERAL_DATA_MOVEMENT"  # mov, lea, xchg
    STACK_MANAGEMENT = "STACK_MANAGEMENT"  # push, pop, enter, leave
    STRING_MANIPULATION = "STRING_MANIPULATION"  # movs, lods, stos, cmps

    # Arithmetic
    BASIC_ARITHMETIC = "BASIC_ARITHMETIC"  # add, sub, inc, dec
    MULTIPLICATION_DIVISION = "MULTIPLICATION_DIVISION"  # mul, div, imul, idiv
    CARRY_ARITHMETIC = "CARRY_ARITHMETIC"  # adc, sbb

    # Logical
    BITWISE_LOGIC = "BITWISE_LOGIC"  # and, or, xor, not
    CONDITIONAL_LOGIC = "CONDITIONAL_LOGIC"  # test, cmp, setX

    # Control Flow
    UNCONDITIONAL_JUMP = "UNCONDITIONAL_JUMP"  # jmp
    CONDITIONAL_JUMP = "CONDITIONAL_JUMP"  # je, jne, jl, jg, etc
    FUNCTION_CONTROL = "FUNCTION_CONTROL"  # call, ret
    LOOPING = "LOOPING"  # loop, loopz, loopnz

    # System
    SYSTEM_CALLS = "SYSTEM_CALLS"  # syscall, int, sysenter
    PRIVILEGED_INSTRUCTIONS = "PRIVILEGED_INSTRUCTIONS"  # hlt, cli, sti
    CPU_FEATURES = "CPU_FEATURES"  # cpuid, rdtsc

    # SIMD & FPU
    SSE_SIMD = "SSE_SIMD"  # SSE instructions
    AVX_SIMD = "AVX_SIMD"  # AVX instructions
    BASIC_FPU = "BASIC_FPU"  # fld, fst, fstp
    FPU_ARITHMETIC = "FPU_ARITHMETIC"  # fadd, fsub, etc

    # Bit Operations
    SHIFT_ROTATE = "SHIFT_ROTATE"  # shl, shr, rol, ror
    BIT_TEST_MODIFY = "BIT_TEST_MODIFY"  # bt, bts, btr, btc

    # Special
    CRYPTOGRAPHIC_OPS = "CRYPTOGRAPHIC_OPS"  # aesenc, aesdec, sha1rnds4
    MISC_OPS = "MISC_OPS"  # nop, ud2, etc


# class BranchType(Enum):
#     DIRECT = "DIRECT"
#     CONDITIONAL = "CONDITIONAL"
#     CALL = "CALL"
#     RETURN = "RETURN"
#     FALLTHROUGH = "FALLTHROUGH"
#     UNKNOWN = "UNKNOWN"


class ApiCategory(Enum):
    FILE_OP = [
        "CreateFile",
        "ReadFile",
        "WriteFile",
        "DeleteFile",
        "SetFilePointer",
        "CopyFile",
        "MoveFile",
        "FindFirstFile",
        "FindNextFile",
    ]
    MEMORY_OP = [
        "VirtualAlloc",
        "VirtualFree",
        "HeapAlloc",
        "HeapFree",
        "LocalAlloc",
        "GlobalAlloc",
        "MapViewOfFile",
        "VirtualProtect",
    ]
    NETWORK_OP = [
        "socket",
        "connect",
        "bind",
        "send",
        "recv",
        "WSAStartup",
        "InternetOpen",
        "InternetConnect",
        "HttpOpenRequest",
        "HttpSendRequest",
        "InternetReadFile",
        "URLDownloadToFile",
    ]
    REGISTRY_OP = [
        "RegOpenKey",
        "RegCreateKey",
        "RegSetValue",
        "RegQueryValue",
        "RegDeleteKey",
        "RegEnumKey",
        "RegFlushKey",
    ]
    PROCESS_OP = [
        "CreateProcess",
        "OpenProcess",
        "TerminateProcess",
        "GetProcessId",
        "CreateProcessAsUser",
        "NtCreateProcess",
    ]
    THREAD_OP = [
        "CreateThread",
        "SuspendThread",
        "ResumeThread",
        "CreateRemoteThread",
        "SetThreadContext",
        "GetThreadContext",
    ]
    INJECTION_OP = [
        "WriteProcessMemory",
        "VirtualAllocEx",
        "NtWriteVirtualMemory",
        "SetWindowsHookEx",
        "QueueUserAPC",
        "NtMapViewOfSection",
    ]
    EVASION_OP = [
        "IsDebuggerPresent",
        "CheckRemoteDebuggerPresent",
        "NtQueryInformationProcess",
        "GetTickCount",
        "OutputDebugString",
        "Sleep",
        "QueryPerformanceCounter",
    ]
    SPYING_OP = [
        "GetAsyncKeyState",
        "GetKeyboardState",
        "GetKeyState",
        "GetForegroundWindow",
        "SetWindowsHookEx",
        "BitBlt",
        "GetClipboardData",
    ]
    SYSTEM_OP = [
        "CreateToolhelp32Snapshot",
        "EnumDeviceDrivers",
        "EnumProcesses",
        "GetSystemDirectoryA",
        "GetLogicalDrives",
    ]
    SERVICE_OP = [
        "CreateServiceA",
        "OpenServiceA",
        "StartServiceA",
        "DeleteService",
        "OpenSCManagerA",
        "ControlService",
    ]
    CRYPTO_OP = [
        "CryptAcquireContext",
        "CryptGenKey",
        "CryptEncrypt",
        "CryptDecrypt",
        "CryptCreateHash",
        "CryptHashData",
        "CryptGenRandom",
    ]
    DLL_OP = ["LoadLibrary", "GetProcAddress", "FreeLibrary", "LdrLoadDll"]
    UNKNOWN_OP = []

    @classmethod
    def from_api(cls, api_name):
        for category in cls:
            if any(api_name.startswith(api) for api in category.value):
                return category
        return cls.UNKNOWN_OP


class BinjaDecompiler:
    """
    A Binary Ninja-based decompiler that replicates the functionality of GhidraDecompilerScript.

    This class extracts decompiled code, disassembly with multiple normalization levels,
    and control flow graph information from binary files.
    """

    MIN_FUNCTION_SIZE = 10  # instructions
    MIN_BLOCK_SIZE = 4  # instructions
    INVALID_STACK_SIZE = -1

    def __init__(
        self,
        filepath,
        timeout,
        log=None,
        exporters=None,
        index_prefix=None,
        elastic_index=None,
        known_benign=False,
        known_malicious=False,
        filetype=None,
        sha256=None,
        sha1=None,
        md5=None,
    ):
        """
        Initialize the Binary Ninja decompiler.

        Args:
            filepath: Path to the binary file to analyze
            log: Logger object (optional)
            timeout: Maximum time in seconds for analysis (default: 1200)
            exporters: List of exporters for the results (optional)
            index_prefix: Prefix for elastic index (optional)
            elastic_index: Elastic index name (optional)
            known_benign: Whether the file is known to be benign (optional)
            known_malicious: Whether the file is known to be malicious (optional)
            filetype: Type of the file (optional)
            sha256: SHA256 hash of the file (optional)
            sha1: SHA1 hash of the file (optional)
            md5: MD5 hash of the file (optional)
        """
        self.filepath = filepath
        self.log = log if log else self._setup_default_logger()
        self.BINJA_TIMEOUT = timeout
        self.bv = None
        self.analysis_results = None
        self.errors = []
        self.sha256 = sha256
        self.sha1 = sha1
        self.md5 = md5
        self.exporters = exporters
        self.index_prefix = index_prefix
        self.elastic_index = elastic_index
        self.known_benign = known_benign
        self.known_malicious = known_malicious
        self.filetype = filetype

        # Map to track instruction categorization
        self.opcode_categories = self._initialize_opcode_categories()
        mainthread.set_worker_thread_count(2)

        binaryninja.Settings().set_integer(
            "rendering.strings.maxAnnotationLength", 100000
        )
        binaryninja.Settings().set_integer("analysis.limits.maxStringLength", 100000)

    def _setup_default_logger(self):
        """Create a simple default logger if none is provided."""
        import logging

        logger = logging.getLogger("BinjaDecompiler")
        logger.setLevel(logging.INFO)

        # Create console handler
        ch = logging.StreamHandler()
        ch.setLevel(logging.INFO)

        # Create formatter
        formatter = logging.Formatter(
            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
        )
        ch.setFormatter(formatter)

        # Add handler to logger
        logger.addHandler(ch)

        return logger

    def log_error(
        self, message, function_name, address, exception=None, error_location="unknown"
    ):
        """Log an error during processing."""
        error_msg = f"Error in function {function_name} at {address}: {message}"
        if exception:
            error_msg += f" - {str(exception)}"

        self.log.error(error_msg)

        # Add to errors list
        error = {
            "function_name": function_name,
            "function_address": str(address),
            "error_location": error_location,
            "error_message": message,
            "error_details": str(exception) if exception else "",
            "error_type": type(exception).__name__ if exception else "Unknown",
            "timestamp": int(time.time() * 1000),
        }
        self.errors.append(error)

    def calculate_sha256(self, input_str):
        """Calculate SHA256 hash of a string."""
        return hashlib.sha256(input_str.encode("utf-8")).hexdigest()

    # def _initialize_opcode_categories(self):
    #     """Initialize mapping of opcodes to categories."""
    #     opcode_categories = {}

    #     # Data Movement (after string ops to avoid MOVS confusion)
    #     for op in ["MOV", "MOVSX", "MOVZX", "LEA", "XCHG"]:
    #         opcode_categories[op] = InstructionType.GENERAL_DATA_MOVEMENT

    #     # String Operations
    #     for op in ["MOVS", "LODS", "STOS", "CMPS", "SCAS", "REP", "REPE", "REPNE"]:
    #         opcode_categories[op] = InstructionType.STRING_MANIPULATION

    #     # Stack Operations
    #     for op in ["PUSH", "POP", "ENTER", "LEAVE", "PUSHA", "POPA"]:
    #         opcode_categories[op] = InstructionType.STACK_MANAGEMENT

    #     # Control Flow (non-conditional)
    #     for op in ["JMP", "CALL", "RET", "RETN"]:
    #         opcode_categories[op] = InstructionType.FUNCTION_CONTROL

    #     # Conditional Jumps and Loops
    #     for op in [
    #         "JE",
    #         "JNE",
    #         "JG",
    #         "JGE",
    #         "JL",
    #         "JLE",
    #         "JA",
    #         "JAE",
    #         "JB",
    #         "JBE",
    #         "JZ",
    #         "JNZ",
    #         "JS",
    #         "JNS",
    #         "JO",
    #         "JNO",
    #         "JP",
    #         "JNP",
    #     ]:
    #         opcode_categories[op] = InstructionType.CONDITIONAL_JUMP
    #     for op in ["LOOP", "LOOPE", "LOOPNE", "LOOPZ", "LOOPNZ"]:
    #         opcode_categories[op] = InstructionType.LOOPING

    #     # Arithmetic
    #     for op in ["ADD", "SUB", "INC", "DEC"]:
    #         opcode_categories[op] = InstructionType.BASIC_ARITHMETIC
    #     for op in ["MUL", "DIV", "IMUL", "IDIV"]:
    #         opcode_categories[op] = InstructionType.MULTIPLICATION_DIVISION
    #     for op in ["ADC", "SBB", "NEG"]:
    #         opcode_categories[op] = InstructionType.CARRY_ARITHMETIC

    #     # Logical
    #     for op in ["AND", "OR", "XOR", "NOT"]:
    #         opcode_categories[op] = InstructionType.BITWISE_LOGIC
    #     for op in ["TEST", "CMP"]:
    #         opcode_categories[op] = InstructionType.CONDITIONAL_LOGIC
    #     for op in [
    #         "SET" + suffix
    #         for suffix in [
    #             "E",
    #             "NE",
    #             "G",
    #             "GE",
    #             "L",
    #             "LE",
    #             "A",
    #             "AE",
    #             "B",
    #             "BE",
    #             "Z",
    #             "S",
    #             "NS",
    #             "O",
    #             "NO",
    #             "P",
    #             "NP",
    #         ]
    #     ]:
    #         opcode_categories[op] = InstructionType.CONDITIONAL_LOGIC

    #     # Shifts & Rotates
    #     for op in ["SHL", "SHR", "SAR", "SAL", "ROL", "ROR", "RCL", "RCR"]:
    #         opcode_categories[op] = InstructionType.SHIFT_ROTATE

    #     # System & Interrupts
    #     for op in [
    #         "SYSCALL",
    #         "INT",
    #         "SYSENTER",
    #         "SYSEXIT",
    #         "SGDT",
    #         "SIDT",
    #         "SLDT",
    #         "WRMSR",
    #         "RDMSR",
    #     ]:
    #         opcode_categories[op] = InstructionType.SYSTEM_CALLS

    #     # Floating Point - handle with startswith since there are many
    #     # opcode_categories["F"] = InstructionType.BASIC_FPU

    #     # System Information and Random Number Generation
    #     for op in ["PUSHF", "POPF", "CPUID", "RDTSC", "RDRAND", "RDSEED"]:
    #         opcode_categories[op] = InstructionType.CPU_FEATURES

    #     # Cryptography
    #     # Will handle with startswith for AES and SHA

    #     # Bit Test operations
    #     for op in ["BT", "BTS", "BTR", "BTC", "BSF", "BSR"]:
    #         opcode_categories[op] = InstructionType.BIT_TEST_MODIFY

    #     # SIMD operations - will handle with prefix detection

    #     return opcode_categories

    def _initialize_opcode_categories(self):
        """Initialize mapping of opcodes to categories similar to Ghidra's implementation."""
        opcode_categories = {}
        opcode_index = {}  # Add this to mimic Ghidra's opcodeIndex

        # Define common opcodes array similar to Ghidra's COMMON_OPCODES
        COMMON_OPCODES = [
            # Core instructions (tracked individually)
            "MOV",
            "PUSH",
            "POP",
            "LEA",
            "CALL",
            "RET",  # Data movement and control
            "ADD",
            "SUB",
            "MUL",
            "DIV",  # Basic arithmetic
            "AND",
            "OR",
            "XOR",
            "NOT",  # Logical operations
            "JMP",
            "JE",
            "JNE",  # Basic jumps
            "TEST",
            "CMP",  # Comparisons
            # Grouped categories (aggregated tracking)
            "SIMD_MOVE",  # MOVAPS, MOVDQA, MOVDQU, etc.
            "COND_JUMP_EXT",  # Other conditional jumps (JG, JL, JGE, etc.)
            "STRING_OP",  # MOVS, STOS, LODS, SCAS, CMPS
            "STACK_ADV",  # ENTER, LEAVE, PUSHA, POPA
            "ARITHMETIC_ADV",  # IMUL, IDIV, ADC, SBB
            "BIT_OP",  # SHL, SHR, SAR, ROL, ROR, etc.
            "FPU_OP",  # FLD, FST, FADD, etc.
            "SYSTEM_OP",  # SYSCALL, INT, SYSENTER
            "CRYPTO_OP",  # AES*, SHA* instructions
            "MISC_OP",  # Rare but interesting (CPUID, RDTSC, etc.)
        ]

        # Create index map like Ghidra
        for i, opcode in enumerate(COMMON_OPCODES):
            opcode_index[opcode] = i

        # Now categorize opcodes using if/elif/else structure like in Ghidra
        for opcode in COMMON_OPCODES:
            # String Operations (checking these first to avoid MOV confusion)
            if opcode.startswith("MOVS") or opcode in [
                "STOS",
                "LODS",
                "SCAS",
                "CMPS",
                "REP",
                "REPE",
                "REPNE",
            ]:
                opcode_categories[opcode] = "STRING_MANIPULATION"

            # Data Movement (after string ops to avoid MOVS confusion)
            elif opcode.startswith("MOV") or opcode in ["LEA", "XCHG"]:
                opcode_categories[opcode] = "DATA_MOVEMENT"

            # Stack Operations
            elif opcode in ["PUSH", "POP", "ENTER", "LEAVE", "PUSHA", "POPA"]:
                opcode_categories[opcode] = "STACK_MANAGEMENT"

            # Control Flow (non-conditional)
            elif opcode in ["JMP", "CALL", "RET"]:
                opcode_categories[opcode] = "CONTROL_FLOW"

            # Conditional Jumps and Loops
            elif opcode.startswith("J") or opcode.startswith("LOOP"):
                opcode_categories[opcode] = "CONDITIONAL_JUMP"

            # Arithmetic
            elif opcode in [
                "ADD",
                "SUB",
                "MUL",
                "DIV",
                "IMUL",
                "IDIV",
                "ADC",
                "SBB",
                "INC",
                "DEC",
                "NEG",
            ]:
                opcode_categories[opcode] = "ARITHMETIC"

            # Logical
            elif opcode in ["AND", "OR", "XOR", "NOT", "TEST", "CMP"]:
                opcode_categories[opcode] = "LOGICAL"

            # Shifts & Rotates
            elif opcode in ["SHL", "SHR", "SAR", "SAL", "ROL", "ROR", "RCL", "RCR"]:
                opcode_categories[opcode] = "SHIFT_ROTATE"

            # System & Interrupts
            elif opcode in [
                "SYSCALL",
                "INT",
                "SYSENTER",
                "SYSEXIT",
                "SGDT",
                "SIDT",
                "SLDT",
                "WRMSR",
                "RDMSR",
            ]:
                opcode_categories[opcode] = "SYSTEM_CALLS"

            # Floating Point
            elif opcode.startswith("F"):
                opcode_categories[opcode] = "FPU_ARITHMETIC"

            # System Information and Random Number Generation
            elif opcode in ["PUSHF", "POPF", "CPUID", "RDTSC", "RDRAND", "RDSEED"]:
                opcode_categories[opcode] = "CPU_FEATURES"

            # Cryptography
            elif opcode.startswith("AES") or opcode.startswith("SHA"):
                opcode_categories[opcode] = "CRYPTOGRAPHIC"

            # Miscellaneous (including flag operations)
            else:
                opcode_categories[opcode] = "MISC"

        # Additional categorization for opcodes not in COMMON_OPCODES
        # This can be used in the normalize_opcode method

        # Store both maps as instance variables
        # self.opcode_categories = opcode_categories
        self.opcode_index = opcode_index

        return opcode_categories

    def normalize_opcode(self, mnemonic):
        """Normalize opcode similar to Ghidra's normalizeOpcode method."""
        if not mnemonic:
            return None

        normalized = mnemonic.upper().strip()

        # Handle prefixes
        if normalized.startswith(("REP", "REPE", "REPNE")):
            space_pos = normalized.find(" ")
            if space_pos > 0:
                normalized = normalized[space_pos + 1 :]

        # Direct matches first
        if normalized in self.opcode_index:
            return normalized

        # Group categorization for opcodes not in the standard list
        if re.match(r"MOV[AU]PS|MOVDQ[AU]|VMOVDQ[AU]|VMOV[AU]PS", normalized):
            return "SIMD_MOVE"

        if re.match(r"J[GLABE][E]?|JG?[EZSC]|JN[GLABE][E]?|JN[EZSC]", normalized):
            return "COND_JUMP_EXT"

        if re.match(r"MOVS|STOS|LODS|SCAS|CMPS|MOVSB|MOVSW|MOVSD", normalized):
            return "STRING_OP"

        if re.match(r"ENTER|LEAVE|PUSHA|POPA", normalized):
            return "STACK_ADV"

        if re.match(r"IMUL|IDIV|ADC|SBB|NEG|BSWAP", normalized):
            return "ARITHMETIC_ADV"

        if re.match(r"SHL|SHR|SAR|SAL|ROL|ROR|RCL|RCR|BT[SR]?|BSF|BSR", normalized):
            return "BIT_OP"

        if normalized.startswith("F"):
            return "FPU_OP"

        if re.match(r"SYSCALL|INT|SYSENTER|SYSEXIT", normalized):
            return "SYSTEM_OP"

        if re.match(r"AES.*|SHA.*|RDRAND|RDSEED", normalized):
            return "CRYPTO_OP"

        if re.match(r"CPUID|RDTSC|UD2|HLT|PAUSE", normalized):
            return "MISC_OP"

        # Default case - keep original mnemonic
        return normalized

    ### NOT USED ANYMORE ###
    # def _categorize_instruction(self, mnemonic):
    #     """Categorize an instruction by its mnemonic."""
    #     mnemonic = mnemonic.upper()

    #     # Check direct matches first
    #     if mnemonic in self.opcode_categories:
    #         return self.opcode_categories[mnemonic]

    #     # Check prefixes for FPU instructions
    #     if mnemonic.startswith("F"):
    #         return InstructionType.BASIC_FPU

    #     # Check for SSE/AVX instructions
    #     if (
    #         mnemonic.startswith("MM")
    #         or mnemonic.startswith("P")
    #         or "PS" in mnemonic
    #         or "PD" in mnemonic
    #         or "SS" in mnemonic
    #         or "SD" in mnemonic
    #     ):
    #         return InstructionType.SSE_SIMD

    #     # Check for AVX instructions (usually start with V)
    #     if mnemonic.startswith("V"):
    #         return InstructionType.AVX_SIMD

    #     # Check for cryptographic instructions
    #     if mnemonic.startswith("AES") or mnemonic.startswith("SHA"):
    #         return InstructionType.CRYPTOGRAPHIC_OPS

    #     # Conditional jumps that weren't in our direct list
    #     if mnemonic.startswith("J") and mnemonic != "JMP":
    #         return InstructionType.CONDITIONAL_JUMP

    #     # Default fallback
    #     return InstructionType.MISC_OPS

    def open_binary(self):
        """Open the binary file with Binary Ninja and perform initial analysis."""
        try:
            # Open the binary
            self.log.info(f"Opening binary file: {self.filepath}")
            self.bv = binaryninja.load(self.filepath, update_analysis=False)

            if self.bv is None:
                self.log.error(f"Failed to open file: {self.filepath}")
                return False

            # Wait for initial analysis to complete with timeout
            self.log.info("Waiting for analysis to complete...")
            analysis_complete = self._wait_for_analysis()

            if not analysis_complete:
                self.log.error("Analysis timed out")
                # Close the binary view if analysis times out
                self.bv.file.close()
                self.bv = None
                return False

            self.log.info(
                f"Analysis completed. Found {len(list(self.bv.functions))} functions."
            )
            return True

        except Exception as e:
            self.log.error(f"Error opening binary: {str(e)}")
            # Make sure to close the binary view if an exception occurs
            if hasattr(self, "bv") and self.bv is not None:
                self.bv.file.close()
                self.bv = None
            return False

    def _wait_for_analysis(self):
        """Wait for Binary Ninja analysis to complete with timeout."""
        start_time = time.time()
        self.bv.update_analysis()

        while (
            self.bv.analysis_progress.state != binaryninja.enums.AnalysisState.IdleState
        ):
            if time.time() - start_time > self.BINJA_TIMEOUT:
                return False
            time.sleep(0.5)

        # Request an additional update after analysis is complete to ensure IL generation
        self.bv.update_analysis_and_wait()
        self.log.debug(
            f"Binja analysis complete: {len(list(self.bv.functions))} functions"
        )

        return True

    def extract(self):
        """
        Extract and process all analysis results.

        Returns:
            bool: True if extraction was successful, False otherwise
        """
        self.log.info(f"Starting binary analysis on {self.filepath}")
        try:
            results = self.analyze_binary()
            if not results:
                self.log.error("Analysis failed to produce results")
                return False

            self.analysis_results = results
            self.log.info(
                f"Successfully analyzed binary: {len(results['decompiled'])} decompiled functions, "
                f"{len(results['disassembled'])} disassembled functions, "
                f"{len(results['cfg'])} basic blocks, "
                f"{len(results['errors'])} errors"
            )
            return True

        except Exception as e:
            self.log.error(f"Error in extraction: {str(e)}")
            return False

    def tag(self):
        """Return the tag for this extractor."""
        return "DECOMPILED"

    def analyze_binary(self):
        """Run Binary Ninja analysis and return results."""
        if not self.open_binary():
            return None

        try:
            # Initialize output structure
            results = {
                "sha256": self.sha256,
                "sha1": self.sha1,
                "md5": self.md5,
                "decompiled": [],
                "disassembled": [],
                "cfg": [],
                "errors": [],
            }

            # Process each function with timeout protection
            function_list = sorted(list(self.bv.functions), key=lambda f: f.start)
            for function in function_list:
                # Skip external and thunk functions
                if (
                    function.symbol.type
                    == binaryninja.enums.SymbolType.ImportedFunctionSymbol
                    or function.is_thunk
                ):
                    continue
                try:
                    # Use a timer to implement function-level timeout
                    function_timeout = 60  # 1 minute per function
                    timer = threading.Timer(
                        function_timeout, self._function_timeout_handler
                    )

                    timer.start()
                    try:
                        # Process both decompiled and disassembled code for the same function
                        decompiled_json = self.extract_decompiled(function)
                        disassembled_json = self.extract_disassembly(function)

                        # If we have both results, add cross-references
                        if decompiled_json and disassembled_json:
                            # Add cross-references
                            decompiled_json["disassembled_function_hash"] = (
                                disassembled_json["disassembled_function_hash"]
                            )
                            disassembled_json["decompiled_function_hash"] = (
                                decompiled_json["decompiled_function_hash"]
                            )

                            # Add to results
                            results["decompiled"].append(decompiled_json)
                            results["disassembled"].append(disassembled_json)
                        elif decompiled_json:
                            # Only decompiled available
                            decompiled_json["disassembled_function_hash"] = None
                            results["decompiled"].append(decompiled_json)
                        elif disassembled_json:
                            # Only disassembled available
                            disassembled_json["decompiled_function_hash"] = None
                            results["disassembled"].append(disassembled_json)

                        # Process CFG
                        # for block in function.basic_blocks:
                        # function_cfg_json = self.extract_function_cfg(function)
                        # if function_cfg_json:
                        #     results["cfg"].append(function_cfg_json)
                        # block_list = sorted(list(function.basic_blocks), key=lambda b: b.start)
                        # for block in block_list:
                        #     block_json = self.extract_basic_block(function, block)
                        #     if block_json:
                        #         results["cfg"].append(block_json)
                    finally:
                        timer.cancel()

                except Exception as e:
                    self.log_error(
                        f"Failed to process function",
                        function.name,
                        function.start,
                        e,
                        "analyze_binary",
                    )

            # Add collected errors
            results["errors"] = self.errors

            return results

        except Exception as e:
            self.log.error(f"Error in binary analysis: {str(e)}")
            return None

        finally:
            # Clean up Binary Ninja resources
            self.cleanup()

    def _function_timeout_handler(self):
        """Handler for function-level timeout."""
        # This is called when a function takes too long to process
        # In a real implementation, you might need a more sophisticated way
        # to interrupt the current function processing
        self.log.warning("Function processing timeout")
        # In reality, it's difficult to gracefully cancel a function analysis in progress
        # You may need to use a more complex approach depending on your specific needs

    def extract_decompiled(self, function):
        """Extract decompiled function code."""
        try:
            decompiled_code = None
            try:
                # Get decompiled code
                function_prototype = str(function)
                decompiled_code = str(function.hlil)
                if not decompiled_code or decompiled_code.strip() == "":
                    self.log.warning(f"Empty decompilation result for {function.name}")
                    return None

                # indented_hlil = textwrap.indent(decompiled_code, '  ')
                # decompiled_code = function_prototype + "\n{\n" + indented_hlil + "\n}"
            except Exception as e:
                self.log.warning(
                    f"Failed to get HLIL for {function.name} at {function.start}: {str(e)}"
                )

            if decompiled_code:
                # Create json object
                function_json = {
                    "decompiled_function_hash": self.calculate_sha256(decompiled_code),
                    "decompiled_function": decompiled_code,
                    "decompiled_function_name": function.name,
                    "decompiled_function_prototype": function_prototype,
                    "decompiled_function_address": function.start,
                    "function_type": self.determine_function_type(function),  # TODO
                }
                return function_json
            else:
                return None

        except Exception as e:
            self.log_error(
                "Failed in decompilation",
                function.name,
                function.start,
                e,
                "extract_decompiled",
            )
            return None

    def determine_function_type(self, function):
        """Determine the type of a function (EXTERNAL, THUNK, LIBRARY, USER)."""
        try:
            symbol_type = function.symbol.type
            # Check if it's an external function (there should not be, already skipping these)
            if symbol_type == binaryninja.enums.SymbolType.ImportedFunctionSymbol:
                return "EXTERNAL"

            # Library functions detected by Binary Ninja
            if symbol_type == binaryninja.enums.SymbolType.LibraryFunctionSymbol:
                return "LIBRARY"

            # Check if it's a thunk function
            if function.is_thunk:
                return "THUNK"

            # Additional library heuristics for cases Binary Ninja might miss
            # Check if it's a library function by name pattern or location
            name = function.name
            if (
                name.startswith("std::")  # C++ standard library
                or name.startswith("__security_")  # MSVC security features
                or name.startswith("__chk")  # Stack/buffer checking
                # or (len(name) > 10 and "@@" in name)  # Complex mangled names
                or name.startswith("operator")  # C++ operators
                or name.startswith("__crt")  # C runtime
                or (
                    name.startswith("??")
                    and len(name) > 20
                    and ("std" in name or "basic_" in name)
                )  # REPLACE mangling check
            ):
                return "LIBRARY"

            # Default to USER function
            return "USER"

        except Exception as e:
            self.log_error(
                "Failed to determine function type",
                function.name,
                function.start,
                e,
                "determine_function_type",
            )
            return "UNKNOWN"

    def extract_disassembly(self, function):
        """Extract disassembled function code with different normalization levels."""
        try:
            # Create settings
            # settings = DisassemblySettings()
            # settings.set_option(DisassemblyOption.ShowAddress, True)
            # settings.set_option(DisassemblyOption.ShowOpcode, True)
            # settings.set_option(DisassemblyOption.WaitForIL, True)

            # Count instructions to check minimum size
            instructions_count = sum(1 for _ in function.instructions)
            if instructions_count < self.MIN_FUNCTION_SIZE:
                return None

            # Build disassembly string and normalized versions
            disassembly_builder = [[], []]  # Address and instruction text
            normalized_builders = [[], [], []]  # One for each normalization level

            # Create a dictionary mapping addresses to instruction tokens
            instr_tokens_by_addr = {}
            for instr_tokens, addr in function.instructions:
                instr_tokens_by_addr[addr] = instr_tokens

            addresses = sorted(instr_tokens_by_addr.keys())
            for address in addresses:
                # Original disassembly with addresses
                # instr_tokens, address = instruction
                instr_tokens = instr_tokens_by_addr[address]
                disassembly_builder[0].append(address)
                disassembly_builder[1].append("".join(map(str, instr_tokens)))

                # Get normalized versions
                try:
                    normalized_versions = self.normalize_instruction_all_levels(
                        instr_tokens
                    )
                    for i in range(3):
                        normalized_builders[i].append(normalized_versions[i])
                except Exception as e:
                    self.log_error(
                        "Failed to normalize instruction",
                        function.name,
                        address,
                        e,
                        "extract_disassembly",
                    )

            # Join with newlines
            disassembly_str = "\n".join(disassembly_builder[1])
            disassembly_with_addresses = "\n".join(
                f"{hex(address)}: {instr_text}"
                for address, instr_text in zip(
                    disassembly_builder[0], disassembly_builder[1]
                )
            )
            normalized_strs = ["\n".join(builder) for builder in normalized_builders]

            # Create disassembly JSON
            disassembly_json = {
                "disassembled_function_hash": self.calculate_sha256(disassembly_str),
                "fully_normalized_disassembly_hash": self.calculate_sha256(
                    normalized_strs[0]
                ),
                "api_normalized_disassembly_hash": self.calculate_sha256(
                    normalized_strs[1]
                ),
                "category_normalized_disassembly_hash": self.calculate_sha256(
                    normalized_strs[2]
                ),
                "disassembled_function": disassembly_with_addresses,
                "disassembled_function_no_addresses": disassembly_str,
                "fully_normalized_disassembly": normalized_strs[0],
                "api_normalized_disassembly": normalized_strs[1],
                "category_normalized_disassembly": normalized_strs[2],
                "disassembled_function_name": function.name,
                "disassembled_function_address": function.start,
                "instructions_count": instructions_count,
                "function_type": self.determine_function_type(function),
            }

            # Add additional metrics
            type_frequencies = self.collect_instruction_types(function)
            disassembly_json["instructions_types"] = list(type_frequencies.keys())
            disassembly_json["control_flow_count"] = (
                self.count_control_flow_instructions(function)
            )
            disassembly_json["memory_access_pattern"] = self.collect_memory_patterns(
                function
            )
            disassembly_json["register_usage"] = self.collect_register_usage(function)
            disassembly_json["data_references_count"] = self.count_data_references(
                function
            )
            disassembly_json["max_block_size"] = self.compute_max_block_size(function)
            disassembly_json["num_calls"] = self.compute_num_calls(function)
            disassembly_json["stack_size"] = self.estimate_stack_size(function)

            return disassembly_json

        except Exception as e:
            self.log_error(
                "Fatal error in disassembly extraction",
                function.name,
                function.start,
                e,
                "extract_disassembly",
            )
            return None

    def normalize_instruction_all_levels(self, instr_tokens):
        """Normalize an instruction at three different levels of abstraction.
        Returns [fully_normalized, api_normalized, category_normalized]
        """
        try:
            # First, extract the mnemonic and operands in their original form
            mnemonic = None
            operands = []
            current_operand = []
            in_operand_list = False

            for token in instr_tokens:
                if token.type == InstructionTextTokenType.InstructionToken:
                    mnemonic = token.text.upper()
                    in_operand_list = True  # After mnemonic, operands follow
                elif in_operand_list:
                    if token.type == InstructionTextTokenType.OperandSeparatorToken:
                        if current_operand:
                            operand_text = "".join(
                                token.text for token in current_operand
                            ).strip()
                            operands.append(operand_text)
                            current_operand = []
                    else:
                        current_operand.append(token)

            # Add the last operand if there is one
            if current_operand:
                operand_text = "".join(token.text for token in current_operand).strip()
                operands.append(operand_text)

            # Process each operand at each level of normalization
            normalized_operands = []
            for operand in operands:
                operand_at_levels = []
                operand_at_levels.append(
                    self.normalize_operand_level0(operand)
                )  # Most abstract
                operand_at_levels.append(
                    self.normalize_operand_level1(operand)
                )  # Medium abstraction
                operand_at_levels.append(
                    self.normalize_operand_level2(operand)
                )  # Least abstract
                normalized_operands.append(operand_at_levels)

            # Create the normalized instruction representations
            normalized_instructions = ["", "", ""]
            if mnemonic:
                for level in range(3):
                    normalized_instruction = mnemonic
                    for operand_at_levels in normalized_operands:
                        normalized_instruction += f" {operand_at_levels[level]}"

                    normalized_instructions[level] = normalized_instruction

                # Add <TARGET> to control flow instructions
                if self.is_control_flow_instruction_by_mnemonic(mnemonic):
                    for level in range(3):
                        normalized_instructions[level] += " <TARGET>"
            else:
                # Fallback if no mnemonic found
                return ["UNKNOWN", "UNKNOWN", "UNKNOWN"]

            return normalized_instructions

        except Exception as e:
            self.log.error(f"Error normalizing instruction: {str(e)}")
            # Fallback on error
            if mnemonic:
                return [mnemonic, mnemonic, mnemonic]
            else:
                return ["UNKNOWN", "UNKNOWN", "UNKNOWN"]

    def normalize_operand_level0(self, operand):
        """Normalize an operand to the most abstract level (level 0)."""
        if operand is None:
            return "UNKNOWN"  # Handle None operands

        operand = operand.strip().upper()

        # Registers - categorize by type (more specific than just "REG")
        if self.is_register(operand):
            return self.normalize_register(operand)

        # Memory operands
        if "[" in operand and "]" in operand:
            return "MEM"

        # Constants/immediate values
        if (
            operand.startswith("0X")
            or operand.isdigit()
            or (operand.startswith("-") and operand[1:].isdigit())
        ):
            return "CONST"

        # Data references/symbols
        if "_" in operand or operand.startswith("0X") or operand.startswith("0x"):
            return "DATA_REF"

        # Default - keep as is if we can't categorize
        return operand

    def normalize_operand_level1(self, operand):
        """Normalize an operand to a medium abstraction level (level 1)."""
        if operand is None:
            return "UNKNOWN"  # Handle None operands

        operand = operand.strip().upper()

        if self.is_register(operand):
            if re.fullmatch(r"(R\d+[DWB]?|[ABCD][LH]|[RE]?[ABCD]X)", operand):
                return "GPR_DATA"
            elif re.fullmatch(r"(R?SI|[RE]?SI|R?DI|[RE]?DI)", operand):
                return "GPR_INDEX"
            elif re.fullmatch(r"(R?SP|[RE]?SP|R?BP|[RE]?BP)", operand):
                return "GPR_STACK"
            elif re.fullmatch(r"XMM\d+", operand):
                return "XMM_REG"
            elif re.fullmatch(r"ST\d+", operand):
                return "FPU_REG"
            else:
                return "OTHER_REG"

        # Memory operands
        if "[" in operand and "]" in operand:
            # mem_content = re.search(r"\[(.*?)\]", operand).group(1)
            # Safely extract memory content
            match = re.search(r"\[(.*?)\]", operand)
            if match:  # Check if the regex match was successful
                mem_content = match.group(1)
                if any(
                    reg in mem_content.upper()
                    for reg in ["SP", "BP", "ESP", "EBP", "RSP", "RBP"]
                ):
                    return "MEM_STACK"
                elif any(
                    reg in mem_content.upper()
                    for reg in ["SI", "DI", "ESI", "EDI", "RSI", "RDI"]
                ):
                    return "MEM_STRING"
                else:
                    return "MEM_GENERAL"
            else:
                return "MEM_UNKNOWN"  # Fallback if regex doesn't match

        # Constants/immediate values
        if (
            operand.startswith("0X")
            or operand.isdigit()
            or (operand.startswith("-") and operand[1:].isdigit())
        ):
            try:
                value = int(operand, 0 if operand.startswith("0X") else 10)
                if -16 <= value <= 16:
                    return f"CONST_{value}"
                else:
                    return "CONST_LARGE"
            except ValueError:
                return "CONST"

        # Data references/symbols
        if "_" in operand:
            if self.is_likely_api(operand):
                api_name = self.resolve_api_name(operand)
                if api_name:
                    return f"API_{api_name}"
            return "DATA_SYM"

        # Default - keep as is if we can't categorize
        return operand

    def normalize_operand_level2(self, operand):
        """Normalize an operand to the category level (level 2)."""
        if operand is None:
            return "UNKNOWN"  # Handle None operands

        operand = operand.strip().upper()

        # Register categorization
        if self.is_register(operand):
            if operand == "RSP":
                return "REG_64_SP"
            elif operand == "RBP":
                return "REG_64_BP"
            elif re.fullmatch(r"R\d+", operand) or operand in [
                "RAX",
                "RBX",
                "RCX",
                "RDX",
                "RSI",
                "RDI",
                "RBP",
                "RSP",
                "RIP",
            ]:
                return "REG_64"
            elif re.fullmatch(r"(E[ABCD]X|ESI|EDI|EBP|ESP|EIP|R\d+D)", operand):
                return "REG_32"
            elif re.fullmatch(r"[ABCD][XHL]|SI|DI|SP|BP|IP|.*W", operand):
                return "REG_16_8"
            else:
                return "REG_SPECIAL"

        # Memory operand categorization
        if "[" in operand and "]" in operand:
            # mem_content = re.search(r"\[(.*?)\]", operand).group(1)
            # Safely extract memory content
            match = re.search(r"\[(.*?)\]", operand)
            if match:  # Check if the regex match was successful
                mem_content = match.group(1)
                if "+" in mem_content and "*" in mem_content:
                    return "MEM_SCALED_INDEX"
                elif "+" in mem_content or "-" in mem_content:
                    return "MEM_BASE_OFFSET"
                else:
                    return "MEM_DIRECT"
            else:
                return "MEM_UNKNOWN"  # Fallback if regex doesn't match

        # Constants
        if operand.startswith("0X"):
            return "CONST_HEX"
        elif re.fullmatch(r"-?\d+", operand):
            return "CONST_DEC"

        # Symbol/data reference
        if "_" in operand:
            if self.is_likely_api(operand):
                api_name = self.resolve_api_name(operand)
                if api_name:
                    api_category = ApiCategory.from_api(api_name).name
                    return f"API_{api_category}"
            return "DATA_OTHER"

        # Default
        return operand

    def is_control_flow_instruction_by_mnemonic(self, mnemonic):
        """Check if an instruction is a control flow instruction based on its mnemonic."""
        if not mnemonic:
            return False

        mnemonic = mnemonic.upper()
        return (
            mnemonic.startswith("J")  # All jumps (JMP, JE, JNE, etc.)
            or mnemonic == "CALL"  # Function calls
            or mnemonic == "RET"  # Return
            or mnemonic == "RETN"  # Another form of return
            or mnemonic.startswith("LOOP")
        )  # Loop instructions

    def is_register(self, operand):
        """Check if an operand is a register."""
        # Strip any whitespace and commas
        operand = operand.strip().replace(",", "").upper()

        # Common x86_64 registers
        common_registers = {
            # 64-bit general purpose
            "RAX",
            "RBX",
            "RCX",
            "RDX",
            "RSI",
            "RDI",
            "RBP",
            "RSP",
            "R8",
            "R9",
            "R10",
            "R11",
            "R12",
            "R13",
            "R14",
            "R15",
            "RIP",
            # 32-bit general purpose
            "EAX",
            "EBX",
            "ECX",
            "EDX",
            "ESI",
            "EDI",
            "EBP",
            "ESP",
            "EIP",
            "R8D",
            "R9D",
            "R10D",
            "R11D",
            "R12D",
            "R13D",
            "R14D",
            "R15D",
            # 16-bit general purpose
            "AX",
            "BX",
            "CX",
            "DX",
            "SI",
            "DI",
            "BP",
            "SP",
            "IP",
            "R8W",
            "R9W",
            "R10W",
            "R11W",
            "R12W",
            "R13W",
            "R14W",
            "R15W",
            # 8-bit general purpose
            "AL",
            "BL",
            "CL",
            "DL",
            "AH",
            "BH",
            "CH",
            "DH",
            "R8B",
            "R9B",
            "R10B",
            "R11B",
            "R12B",
            "R13B",
            "R14B",
            "R15B",
            # SIMD
            "XMM0",
            "XMM1",
            "XMM2",
            "XMM3",
            "XMM4",
            "XMM5",
            "XMM6",
            "XMM7",
            "XMM8",
            "XMM9",
            "XMM10",
            "XMM11",
            "XMM12",
            "XMM13",
            "XMM14",
            "XMM15",
            "YMM0",
            "YMM1",
            "YMM2",
            "YMM3",
            "YMM4",
            "YMM5",
            "YMM6",
            "YMM7",
            "YMM8",
            "YMM9",
            "YMM10",
            "YMM11",
            "YMM12",
            "YMM13",
            "YMM14",
            "YMM15",
            # FPU
            "ST0",
            "ST1",
            "ST2",
            "ST3",
            "ST4",
            "ST5",
            "ST6",
            "ST7",
            # Segment
            "CS",
            "DS",
            "ES",
            "FS",
            "GS",
            "SS",
            # Control
            "CR0",
            "CR2",
            "CR3",
            "CR4",
            "CR8",
            # Debug
            "DR0",
            "DR1",
            "DR2",
            "DR3",
            "DR6",
            "DR7",
        }

        # Check exact matches first
        if operand in common_registers:
            return True

        # Check register patterns (R8-R15 variants not in our list)
        if re.match(r"R\d+[DWBX]?|XMM\d+|YMM\d+|ST\d+|DR\d+|CR\d+", operand):
            return True

        return False

    def is_likely_api(self, operand):
        """Check if an operand is likely an API call reference."""
        # Simple heuristic: contains function-like name with common API prefixes
        common_prefixes = [
            "Create",
            "Get",
            "Set",
            "Open",
            "Close",
            "Read",
            "Write",
            "Alloc",
            "Free",
            "Init",
            "Reg",
            "Virt",
            "Heap",
            "Mem",
            "File",
            "Str",
            "Net",
            "Http",
            "Socket",
        ]

        # API calls are often referenced through memory operations or as data references
        if any(prefix in operand for prefix in common_prefixes) and (
            "_" in operand or "[" in operand
        ):
            return True
        return False

    def resolve_api_name(self, operand):
        """Resolve the API name from an operand."""
        # Extract potential API name from memory reference or symbol
        if "_" in operand:
            # Extract from symbol name like "call _CreateFileW"
            parts = operand.split("_", 1)
            if len(parts) > 1:
                # Strip common prefixes like 'imp.' or suffixes like '@IAT'
                api_name = parts[1].split("@")[0].split(".")[0]
                return api_name
        elif "[" in operand and "]" in operand:
            # Extract from memory reference
            mem_content = re.search(r"\[(.*?)\]", operand).group(1)
            if "_" in mem_content:
                parts = mem_content.split("_", 1)
                if len(parts) > 1:
                    api_name = parts[1].split("@")[0].split(".")[0]
                    return api_name

        # If we couldn't extract a likely API name
        return None

    def is_control_flow_instruction(self, instr_tokens):
        """Check if an instruction is a control flow instruction (jump, call, return, loop)."""
        try:
            # Extract the mnemonic from the instruction tokens
            mnemonic = None
            for token in instr_tokens:
                if token.type == InstructionTextTokenType.InstructionToken:
                    mnemonic = token.text.upper()
                    break

            if not mnemonic:
                return False

            # Check if it's a jump, call, return, or loop instruction
            return (
                mnemonic.startswith("J")  # All jumps (JMP, JE, JNE, etc.)
                or mnemonic == "CALL"  # Function calls
                or mnemonic == "RET"  # Return
                or mnemonic == "RETN"  # Another form of return
                or mnemonic.startswith("LOOP")
            )  # Loop instructions

        except Exception:
            # If we can't determine, assume it's not a control flow instruction
            return False

    def __enter__(self):
        """Context manager entry point."""
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        """Context manager exit point - clean up resources."""
        self.cleanup()

    def cleanup(self):
        """Clean up resources used by the decompiler."""
        try:
            if self.bv:
                # Make sure to cancel any pending analysis
                if hasattr(self.bv, "abort_analysis"):
                    self.bv.abort_analysis()
                self.bv.file.close()
                self.bv = None

                # # Explicitly shut down Binary Ninja
                # try:
                #     #import binaryninja
                #     binaryninja.shutdown()
                #     self.log.info("Binary Ninja shutdown successful")
                # except Exception as e:
                #     self.log.error(f"Error during Binary Ninja shutdown: {str(e)}")

            # Force garbage collection
            import gc

            gc.collect()

            self.log.info("Cleanup completed successfully")
        except Exception as e:
            self.log.error(f"Error during cleanup: {str(e)}")

    def get_clickhouse_table(self):
        """Not used directly as we're handling multiple tables."""
        pass

    @staticmethod
    def calculate_tlsh(data):
        """Calculate TLSH fuzzy hash of data if available.

        Args:
            data: The string data to calculate the hash for

        Returns:
            The TLSH hash as a string, or empty string if not available
        """
        try:
            import tlsh

            if len(data) >= 50:  # TLSH requires at least 50 bytes
                return tlsh.hash(data.encode("utf-8"))
        except (ImportError, Exception):
            pass
        return ""

    @staticmethod
    def calculate_ssdeep(data):
        """Calculate ssdeep fuzzy hash of data if available.

        Args:
            data: The string data to calculate the hash for

        Returns:
            The ssdeep hash as a string, or empty string if not available
        """
        try:
            import ppdeep

            if len(data) > 1:
                return ppdeep.hash(data)
        except (ImportError, Exception):
            pass
        return ""

    # def collect_instruction_types(self, function):
    #     """Collect instruction type frequencies from a function.

    #     Args:
    #         function: The Binary Ninja function to analyze

    #     Returns:
    #         A dictionary of instruction type frequencies
    #     """
    #     type_frequencies = {}

    #     try:
    #         # Iterate through all instructions in the function
    #         for instruction in function.instructions:
    #             instr_tokens = instruction[0]  # Get the instruction tokens

    #             # Extract the mnemonic from the instruction tokens
    #             mnemonic = None
    #             for token in instr_tokens:
    #                 if token.type == InstructionTextTokenType.InstructionToken:
    #                     mnemonic = token.text.upper()
    #                     break

    #             if not mnemonic:
    #                 continue

    #             # Categorize instruction based on mnemonic

    #             # Data Movement Categories
    #             if re.match(r"MOV|LEA|XCHG", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.GENERAL_DATA_MOVEMENT.name
    #                 )
    #             elif re.match(r"PUSH|POP|ENTER|LEAVE|PUSHA|POPA", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.STACK_MANAGEMENT.name
    #                 )
    #             elif re.match(r"MOVS|LODS|STOS|CMPS|SCAS|REP.*", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.STRING_MANIPULATION.name
    #                 )

    #             # Arithmetic Categories
    #             elif re.match(r"ADD|SUB|INC|DEC", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.BASIC_ARITHMETIC.name
    #                 )
    #             elif re.match(r"MUL|DIV|IMUL|IDIV", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.MULTIPLICATION_DIVISION.name
    #                 )
    #             elif re.match(r"ADC|SBB", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.CARRY_ARITHMETIC.name
    #                 )

    #             # Logical Categories
    #             elif re.match(r"AND|OR|XOR|NOT", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.BITWISE_LOGIC.name
    #                 )
    #             elif re.match(r"TEST|CMP|SET[A-Z]+", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.CONDITIONAL_LOGIC.name
    #                 )

    #             # Control Flow Categories
    #             elif mnemonic == "JMP":
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.UNCONDITIONAL_JUMP.name
    #                 )
    #             elif re.match(r"J[A-Z]+", mnemonic) and mnemonic != "JMP":
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.CONDITIONAL_JUMP.name
    #                 )
    #             elif re.match(r"CALL|RET", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.FUNCTION_CONTROL.name
    #                 )
    #             elif re.match(r"LOOP.*", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.LOOPING.name
    #                 )

    #             # System Categories
    #             elif re.match(r"SYSCALL|SYSENTER|INT", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.SYSTEM_CALLS.name
    #                 )
    #             elif re.match(r"HLT|CLI|STI", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.PRIVILEGED_INSTRUCTIONS.name
    #                 )
    #             elif re.match(r"CPUID|RDTSC|RDTSCP", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.CPU_FEATURES.name
    #                 )

    #             # SIMD & FPU Categories
    #             elif mnemonic.startswith("V") and any(
    #                 x in mnemonic for x in ["PS", "PD", "SS", "SD"]
    #             ):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.AVX_SIMD.name
    #                 )
    #             elif (
    #                 any(x in mnemonic for x in ["PS", "PD", "SS", "SD"])
    #                 or mnemonic.startswith("MM")
    #                 or mnemonic.startswith("P")
    #             ):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.SSE_SIMD.name
    #                 )
    #             elif re.match(r"FLD|FST|FSTP", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.BASIC_FPU.name
    #                 )
    #             elif re.match(r"FADD|FSUB|FMUL|FDIV|FSQRT", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.FPU_ARITHMETIC.name
    #                 )

    #             # Bit Operation Categories
    #             elif re.match(r"SHL|SHR|SAR|SAL|ROR|ROL|RCR|RCL", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.SHIFT_ROTATE.name
    #                 )
    #             elif re.match(r"BT|BTS|BTR|BTC|BSF|BSR", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.BIT_TEST_MODIFY.name
    #                 )

    #             # Special Categories
    #             elif re.match(r"AES.*|SHA.*|RDRAND|RDSEED", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.CRYPTOGRAPHIC_OPS.name
    #                 )
    #             elif re.match(r"NOP|UD2|INT3", mnemonic):
    #                 self._increment_frequency(
    #                     type_frequencies, InstructionType.MISC_OPS.name
    #                 )

    #     except Exception as e:
    #         self.log_error(
    #             "Failed to collect instruction types",
    #             function.name,
    #             function.start,
    #             e,
    #             "collect_instruction_types",
    #         )

    #     return type_frequencies

    def collect_instruction_types(self, function):
        """Collect instruction type frequencies from a function."""
        type_frequencies = {}

        try:
            # Iterate through all instructions in the function
            for instruction in function.instructions:
                instr_tokens = instruction[0]  # Get the instruction tokens

                # Extract the mnemonic from the instruction tokens
                mnemonic = None
                for token in instr_tokens:
                    if token.type == InstructionTextTokenType.InstructionToken:
                        mnemonic = token.text
                        break

                if not mnemonic:
                    continue

                # Use normalize_opcode to get standardized opcode
                normalized = self.normalize_opcode(mnemonic)

                # Get category from opcode_categories or use the instruction type directly
                category = self.opcode_categories.get(normalized)
                if category:
                    self._increment_frequency(type_frequencies, category)

        except Exception as e:
            self.log_error(
                "Failed to collect instruction types",
                function.name,
                function.start,
                e,
                "collect_instruction_types",
            )

        return type_frequencies

    def _increment_frequency(self, frequencies, type_name):
        """Increment the frequency count for an instruction type."""
        if type_name in frequencies:
            frequencies[type_name] += 1
        else:
            frequencies[type_name] = 1

    def count_control_flow_instructions(self, function):
        """Count the number of control flow instructions in a function."""
        count = 0
        try:
            for instruction in function.instructions:
                instr_tokens = instruction[0]
                if self.is_control_flow_instruction(instr_tokens):
                    count += 1
        except Exception as e:
            self.log_error(
                "Failed to count control flow instructions",
                function.name,
                function.start,
                e,
                "count_control_flow_instructions",
            )
        return count

    def collect_memory_patterns(self, function):
        """Collect memory access patterns from a function."""
        patterns = []
        try:
            for instruction in function.instructions:
                instr_tokens = instruction[0]

                # We need to capture memory operands between BeginMemoryOperandToken and EndMemoryOperandToken
                in_memory_operand = False
                memory_operand_text = ""

                for token in instr_tokens:
                    if token.type == InstructionTextTokenType.BeginMemoryOperandToken:
                        in_memory_operand = True
                        memory_operand_text = ""
                    elif token.type == InstructionTextTokenType.EndMemoryOperandToken:
                        in_memory_operand = False

                        # Process the captured memory operand text
                        if memory_operand_text:
                            # Categorize memory access pattern
                            if (
                                "+" in memory_operand_text
                                and "*" in memory_operand_text
                            ):
                                if "MEM_SCALED_INDEX" not in patterns:
                                    patterns.append("MEM_SCALED_INDEX")
                            elif (
                                "+" in memory_operand_text or "-" in memory_operand_text
                            ):
                                if "MEM_BASE_OFFSET" not in patterns:
                                    patterns.append("MEM_BASE_OFFSET")
                            else:
                                if "MEM_DIRECT" not in patterns:
                                    patterns.append("MEM_DIRECT")

                            # Check for stack accesses
                            if any(
                                reg in memory_operand_text
                                for reg in ["SP", "BP", "ESP", "EBP", "RSP", "RBP"]
                            ):
                                if "MEM_STACK" not in patterns:
                                    patterns.append("MEM_STACK")
                            # Check for string operations
                            elif any(
                                reg in memory_operand_text
                                for reg in ["SI", "DI", "ESI", "EDI", "RSI", "RDI"]
                            ):
                                if "MEM_STRING" not in patterns:
                                    patterns.append("MEM_STRING")
                    elif in_memory_operand:
                        # Accumulate token text while inside a memory operand
                        memory_operand_text += token.text
        except Exception as e:
            self.log_error(
                "Failed to collect memory patterns",
                function.name,
                function.start,
                e,
                "collect_memory_patterns",
            )
        return patterns

    def collect_register_usage(self, function):
        """Collect register usage from a function."""
        registers = []
        try:
            # Define register groups we're interested in tracking
            register_groups = {
                "GPR": [
                    "RAX",
                    "RBX",
                    "RCX",
                    "RDX",
                    "EAX",
                    "EBX",
                    "ECX",
                    "EDX",
                    "AX",
                    "BX",
                    "CX",
                    "DX",
                ],
                "GPR_INDEX": ["RSI", "RDI", "ESI", "EDI", "SI", "DI"],
                "GPR_STACK": ["RSP", "RBP", "ESP", "EBP", "SP", "BP"],
                "SIMD": ["XMM", "YMM", "ZMM"],
                "FPU": ["ST", "ST0", "ST1", "ST2", "ST3", "ST4", "ST5", "ST6", "ST7"],
                "FLAGS": ["FLAGS", "EFLAGS", "RFLAGS"],
                "CONTROL_REGISTER": ["CR0", "CR2", "CR3", "CR4", "CR8"],
                "DEBUG_REGISTER": ["DR0", "DR1", "DR2", "DR3", "DR6", "DR7"],
            }

            # Extract registers from instructions
            for instruction in function.instructions:
                instr_tokens = instruction[0]
                for token in instr_tokens:
                    if token.type == InstructionTextTokenType.RegisterToken:
                        reg = token.text.upper()
                        # Check which group this register belongs to
                        for group, regs in register_groups.items():
                            # if any(r in reg for r in regs) or any(reg.startswith(r) for r in regs):
                            if any(reg == r or reg.startswith(r) for r in regs):
                                if group not in registers:
                                    registers.append(group)
                                break
        except Exception as e:
            self.log_error(
                "Failed to collect register usage",
                function.name,
                function.start,
                e,
                "collect_register_usage",
            )
        return registers

    def count_data_references(self, function):
        """Count the number of data references in a function."""
        count = 0
        try:
            for block in function.mlil:
                for instr in block:
                    instr_str = str(instr)
                    logged = False
                    src = None  # Initialize src to None

                    # Check for constant dereferencing or symbolic refs
                    if hasattr(instr, "src"):
                        src = instr.src
                        if isinstance(
                            src, binaryninja.mediumlevelil.MediumLevelILConstPtr
                        ) or isinstance(
                            src, binaryninja.mediumlevelil.MediumLevelILConst
                        ):
                            # print(f"[{function.name}] Matched CONST in: {instr_str}")
                            count += 1
                            logged = True

                    # Check full string for hardcoded addresses or symbol-like tokens
                    if re.search(r"\b0x[0-9A-Fa-f]{3,}\b", instr_str) and not logged:
                        # print(f"[{function.name}] Matched address literal in: {instr_str}")
                        count += 1
                        logged = True

                    if "_" in instr_str and not logged:
                        # print(f"[{function.name}] Matched symbol-like operand in: {instr_str}")
                        count += 1
                        logged = True

                    # Only check for MediumLevelILConstPtr if src exists
                    if src is not None and isinstance(
                        src, binaryninja.mediumlevelil.MediumLevelILConstPtr
                    ):
                        addr = src.constant
                        # Check if address is in data sections
                        segment = self.bv.get_segment_at(addr)
                        if segment and segment.writable:
                            # print(f"[{function.name}] Matched data section reference in: {instr_str}")
                            count += 1
                            logged = True
        # except Exception as e:
        #     self.log_error(
        #         f"Failed to count data references for {function.name}: {e}",
        #         function.name,
        #         function.start,
        #         e,
        #         "count_data_references",
        #     )
        except Exception as e:
            self.log.warning(
                f"Failed to use MLIL for counting data references in {function.name} at {function.start}: {e}"
            )
            # # Fallback to using raw disassembly if MLIL fails
            # try:
            #     self.log.info(f"Falling back to disassembly for data reference counting in {function.name}")
            #     for instr_tokens, addr in function.instructions:
            #         instr_text = "".join(str(token) for token in instr_tokens)

            #         # Look for hex constants that might be addresses
            #         if re.search(r"\b0x[0-9A-Fa-f]{3,}\b", instr_text):
            #             count += 1
            #             continue

            #         # Look for symbol references
            #         if "_" in instr_text:
            #             count += 1
            #             continue

            #         # Check for memory references
            #         for token in instr_tokens:
            #             if "[" in str(token) and "]" in str(token):
            #                 # Memory reference potentially to data
            #                 count += 1
            #                 break

            #     self.log.info(f"Fallback for {function.name} counted {count} data references")
            # except Exception as e2:
            #     self.log_error(
            #         f"Fallback also failed for data references in {function.name}: {e2}",
            #         function.name,
            #         function.start,
            #         e2,
            #         "count_data_references_fallback",
            #     )
        return count

    def compute_max_block_size(self, function):
        """Compute the maximum basic block size in a function."""
        max_size = 0
        for block in function.basic_blocks:
            try:
                # Count instructions in this block using the direct length approach
                # This avoids UTF-8 decoding issues entirely
                block_size = len(list(block.disassembly_text))
                max_size = max(max_size, block_size)
            except Exception as e:
                self.log_error(
                    f"[HandledError] computing max block size: {e}",
                    function.name,
                    function.start,
                    e,
                    "compute_max_block_size",
                )
        return max_size

    def compute_num_calls(self, function):
        """Compute the number of call instructions in a function."""
        num_calls = 0
        try:
            for instruction in function.instructions:
                instr_tokens = instruction[0]
                # Extract the mnemonic
                for token in instr_tokens:
                    if token.type == InstructionTextTokenType.InstructionToken:
                        if token.text.upper() == "CALL":
                            num_calls += 1
                        break
        except Exception as e:
            self.log_error(
                "Failed to compute number of calls",
                function.name,
                function.start,
                e,
                "compute_num_calls",
            )
        return num_calls

    def estimate_stack_size(self, function):
        """Estimate the stack size used by a function."""
        try:
            # Binary Ninja provides a stack adjustment value for functions
            # Need to convert OffsetWithConfidence to a plain integer
            stack_adjust = function.stack_adjustment
            if hasattr(stack_adjust, "value"):  # Handle OffsetWithConfidence objects
                return stack_adjust.value
            return stack_adjust
        except Exception as e:
            self.log_error(
                "Failed to estimate stack size",
                function.name,
                function.start,
                e,
                "estimate_stack_size",
            )
            return self.INVALID_STACK_SIZE

    def _prepare_for_serialization(self, obj):
        """Recursively prepare an object for JSON serialization by converting Binary Ninja custom types."""
        if isinstance(obj, dict):
            return {
                key: self._prepare_for_serialization(value)
                for key, value in obj.items()
            }
        elif isinstance(obj, list):
            return [self._prepare_for_serialization(item) for item in obj]
        elif hasattr(obj, "value") and hasattr(
            obj, "confidence"
        ):  # OffsetWithConfidence, etc.
            return obj.value
        elif hasattr(obj, "__str__") and not isinstance(
            obj, (str, int, float, bool, type(None))
        ):
            return str(obj)
        else:
            return obj

    def normalize_register(self, register):
        """Normalize a register name to a more generic category."""
        # Convert to uppercase for consistent matching
        register = register.upper()

        # General purpose registers
        if re.match(r"E?[ABCD]X|E?SI|E?DI|R\d+", register):
            return "GPR"

        # Stack/base pointers
        if re.match(r"E?[BS]P|R?SP", register):
            return "PTR"

        # SIMD registers
        if re.match(r"XMM\d+", register):
            return "XMM"

        # FPU registers
        if re.match(r"ST\d+", register):
            return "FPU"

        # Any other register
        return "REG"

    # def extract_function_cfg(self, function):
    #     """Extract information about a function and return it as a dictionary."""
    #     function_address = function.start
    #     function_data = {"function_address": function_address, "blocks": []}

    #     try:
    #         # Second pass: extract block data with graph structure information
    #         for block in function.basic_blocks:
    #             # Build block instructions string
    #             # block_instructions = self.get_block_instructions(block)
    #             block_instructions = "\n".join(
    #                 str(line) for line in block.disassembly_text
    #             )

    #             # Calculate block_id
    #             block_id_input = (
    #                 block_instructions
    #                 + str(block.start)
    #                 + str(block.end)
    #                 + str(function.start)
    #             )
    #             block_id = self.calculate_sha256(block_id_input)

    #             # Get constant references
    #             constants = self.extract_constant_references(block)

    #             # Determine block type
    #             block_type = self.determine_block_type(block)

    #             # Get graph structure information from flow graph node
    #             branch_types = []

    #             # Extract successors directly from basic block
    #             successor_blocks = [edge.target.start for edge in block.outgoing_edges]

    #             # Extract predecessors directly from basic block
    #             predecessor_blocks = [
    #                 edge.source.start for edge in block.incoming_edges
    #             ]

    #             # Determine branch type from outgoing edges
    #             branch_type = self.determine_branch_type(block)

    #             # Add normalized instructions
    #             normalized_builders = ["", "", ""]  # One for each normalization level

    #             # Fix: DisassemblyTextLine is not iterable, we need to access its tokens and addr attributes directly
    #             for line in block.disassembly_text:
    #                 tokens = line.tokens
    #                 addr = line.address

    #                 # Skip non-instruction lines (labels, etc.)
    #                 if not any(
    #                     token.type == InstructionTextTokenType.InstructionToken
    #                     for token in tokens
    #                 ):
    #                     continue

    #                 normalized_versions = self.normalize_instruction_all_levels(tokens)
    #                 for i in range(3):
    #                     normalized_builders[i] += normalized_versions[i]
    #                     if self.is_control_flow_instruction(tokens):
    #                         normalized_builders[i] += " <TARGET>"
    #                     normalized_builders[i] += "\n"

    #             fully_normalized = normalized_builders[0]
    #             api_normalized = normalized_builders[1]
    #             category_normalized = normalized_builders[2]

    #             instructions_count = len(block.disassembly_text)

    #             # # Extract constant references
    #             # referenced_constants = self.extract_constant_references(block)
    #             # if referenced_constants:
    #             #     block_json["referenced_constants"] = referenced_constants

    #             # Create block record
    #             block_json = {
    #                 "block_id": block_id,
    #                 "function_address": function_address,
    #                 "block_start_address": block.start,
    #                 "block_end_address": block.end,
    #                 "block_size": block.end - block.start,
    #                 "instructions_count": instructions_count,
    #                 "block_instructions": block_instructions,
    #                 "fully_normalized_instructions": fully_normalized,
    #                 "api_normalized_instructions": api_normalized,
    #                 "category_normalized_instructions": category_normalized,
    #                 "predecessor_blocks": predecessor_blocks,
    #                 "successor_blocks": successor_blocks,
    #                 "is_entry_block": (block.start == function.start),
    #                 "is_exit_block": any(
    #                     edge.type == BranchType.FunctionReturn
    #                     for edge in block.outgoing_edges
    #                 ),
    #                 "branch_type": branch_type,
    #                 "block_type": block_type,
    #                 "referenced_constants": constants,
    #                 "sign": 1,  # For CollapsingMergeTree
    #             }
    #             function_data["blocks"].append(block_json)

    #         return function_data

    #     except Exception as e:
    #         self.log_error(
    #             "Failed to process basic block",
    #             function.name,
    #             block.start,
    #             e,
    #             "extract_function_cfg",
    #         )
    #         return None

    def extract_basic_block(self, function, block):
        """Extract information about a basic block and return it as a dictionary."""
        function_address = function.start

        try:
            instructions = []
            instructions_with_address = []
            current_addr = block.start
            end_addr = block.end

            # Iterate through the block address range instead of using disassembly_text
            while current_addr < end_addr:
                # Get instruction length at this address
                instr_len = self.bv.get_instruction_length(current_addr)
                if instr_len == 0:
                    instr_len = 1  # Fallback to avoid infinite loop

                try:
                    disasm = self.bv.get_disassembly(current_addr)

                    # Format with both hex and text representation
                    instructions.append(disasm)
                    instructions_with_address.append(f"{current_addr:08x}  {disasm}")
                except UnicodeDecodeError:
                    # If there's a decode error, just use the hex representation
                    instructions.append(f"{current_addr:08x}")
                    instructions_with_address.append(f"{current_addr:08x}")
                except Exception as e:
                    # For any other error, include an error message
                    instructions.append(f"{current_addr:08x}  [Error: {str(e)}]")
                    instructions_with_address.append(
                        f"{current_addr:08x}  [Error: {str(e)}]"
                    )
                # Move to next instruction
                current_addr += instr_len

            block_instructions = "\n".join(instructions)
            block_instructions_with_address = "\n".join(instructions_with_address)

            # Skip empty blocks or blocks with no valid instructions
            if not block_instructions.strip():
                return None

            # Calculate block_id
            block_id_input = (
                block_instructions
                + str(block.start)
                + str(block.end)
                + str(function.start)
            )
            block_id = self.calculate_sha256(block_id_input)

            # Get constant references
            constants = self.extract_constant_references(block)

            # Determine block type
            block_type = self.determine_block_type(block)

            # Extract successors directly from basic block
            successor_blocks = [edge.target.start for edge in block.outgoing_edges]

            # Extract predecessors directly from basic block
            predecessor_blocks = [edge.source.start for edge in block.incoming_edges]

            # Determine branch type from outgoing edges
            branch_type = self.determine_branch_type(block)

            # Add normalized instructions
            normalized_builders = ["", "", ""]  # One for each normalization level

            # Track if we've normalized any instructions
            has_normalized_instructions = False

            # Fix: DisassemblyTextLine is not iterable, we need to access its tokens and addr attributes directly
            try:
                for line in block.disassembly_text:
                    tokens = line.tokens
                    addr = line.address

                    # Skip non-instruction lines (labels, etc.)
                    if not any(
                        token.type == InstructionTextTokenType.InstructionToken
                        for token in tokens
                    ):
                        continue

                    normalized_versions = self.normalize_instruction_all_levels(tokens)
                    for i in range(3):
                        normalized_builders[i] += normalized_versions[i]
                        if self.is_control_flow_instruction(tokens):
                            normalized_builders[i] += " <TARGET>"
                        normalized_builders[i] += "\n"

                    has_normalized_instructions = True
            except Exception as e:
                self.log_error(
                    f"[HandleError] normalizing instructions: {e}",
                    "extract_basic_block",
                    block.start,
                    e,
                )

            # If we didn't successfully normalize any instructions, provide default values
            if not has_normalized_instructions:
                fully_normalized = "UNKNOWN_BLOCK"
                api_normalized = "UNKNOWN_BLOCK"
                category_normalized = "UNKNOWN_BLOCK"
            else:
                fully_normalized = (
                    normalized_builders[0] or "EMPTY_BLOCK"
                )  # Ensure not empty
                api_normalized = (
                    normalized_builders[1] or "EMPTY_BLOCK"
                )  # Ensure not empty
                category_normalized = (
                    normalized_builders[2] or "EMPTY_BLOCK"
                )  # Ensure not empty

            # fully_normalized = normalized_builders[0]
            # api_normalized = normalized_builders[1]
            # category_normalized = normalized_builders[2]

            instructions_count = len(block.disassembly_text)

            # Create block record
            block_json = {
                "block_id": block_id,
                "function_address": function_address,
                "block_start_address": block.start,
                "block_end_address": block.end,
                "block_size": block.end - block.start,
                "instructions_count": instructions_count,
                "block_instructions": block_instructions,
                "block_instructions_with_address": block_instructions_with_address,
                "fully_normalized_instructions": fully_normalized,
                "api_normalized_instructions": api_normalized,
                "category_normalized_instructions": category_normalized,
                "predecessor_blocks": predecessor_blocks,
                "successor_blocks": successor_blocks,
                "is_entry_block": (block.start == function.start),
                "is_exit_block": any(
                    edge.type == BranchType.FunctionReturn
                    for edge in block.outgoing_edges
                ),
                "branch_type": branch_type,
                "block_type": block_type,
                "referenced_constants": constants,
                "sign": 1,  # For CollapsingMergeTree
            }

            return block_json

        except Exception as e:
            self.log_error(
                "Failed to process basic block",
                function.name,
                block.start,
                e,
                "extract_basic_block",
            )
            return None

    def determine_branch_type(self, block):
        """
        Determine the type of a branch at the end of a basic block.
        This combines edge type information with instruction analysis.
        """
        try:
            # If no outgoing edges, it might be a return or terminal block
            if not block.outgoing_edges:
                try:
                    # Check if the last instruction is a return
                    for line in reversed(list(block.disassembly_text)):
                        if line.tokens and any(
                            token.text.lower() in ["ret", "retn"]
                            for token in line.tokens
                        ):
                            return "RETURN"
                    return "UNKNOWN"
                except Exception as e:
                    self.log_error(
                        f"[HandledError] determining branch type: {e}",
                        "determine_branch_type",
                        block.start,
                        e,
                    )
                    return "UNKNOWN"

            # Collect branch types from all outgoing edges
            branch_types = []
            for edge in block.outgoing_edges:
                edge_type = edge.type
                # Map edge type to our branch type enum
                if isinstance(edge_type, str):
                    if edge_type == "IndirectCall":
                        branch_types.append("CALL")
                    else:
                        branch_types.append("UNKNOWN")
                else:
                    # Use our mapping for integer/enum values
                    type_mapping = {
                        BranchType.UnconditionalBranch: "DIRECT",  # 0
                        BranchType.FalseBranch: "CONDITIONAL",  # 1
                        BranchType.TrueBranch: "CONDITIONAL",  # 2
                        BranchType.CallDestination: "CALL",  # 3
                        BranchType.FunctionReturn: "RETURN",  # 4
                        BranchType.SystemCall: "CALL",  # 5
                        BranchType.IndirectBranch: "UNKNOWN",  # 6
                        BranchType.ExceptionBranch: "UNKNOWN",  # 7
                        BranchType.UnresolvedBranch: "UNKNOWN",  # 127
                        BranchType.UserDefinedBranch: "UNKNOWN",  # 128
                    }
                    branch_types.append(type_mapping.get(edge_type, "UNKNOWN"))

            # Determine overall branch type (prioritize CALL > RETURN > CONDITIONAL > DIRECT)
            if "CALL" in branch_types:
                return "CALL"
            elif "RETURN" in branch_types:
                return "RETURN"
            elif "CONDITIONAL" in branch_types:
                return "CONDITIONAL"
            elif "DIRECT" in branch_types:
                return "DIRECT"
            elif len(block.outgoing_edges) == 1:
                return "FALLTHROUGH"

            # If edge analysis was inconclusive, fall back to instruction analysis
            last_instr = None
            try:
                for line in reversed(list(block.disassembly_text)):
                    if line.tokens:
                        last_instr = line
                        break
            except Exception as e:
                self.log_error(
                    f"[HandledError] determining branch type: {e}",
                    "determine_branch_type",
                    block.start,
                    e,
                )

            if last_instr:
                mnemonic = None
                for token in last_instr.tokens:
                    if token.type == InstructionTextTokenType.InstructionToken:
                        mnemonic = token.text.lower()
                        break

                if mnemonic:
                    if mnemonic == "call":
                        return "CALL"
                    elif mnemonic == "jmp":
                        return "DIRECT"
                    elif mnemonic.startswith("j") and mnemonic != "jmp":
                        return "CONDITIONAL"
                    elif mnemonic in ["ret", "retn"]:
                        return "RETURN"

            return "UNKNOWN"

        except Exception as e:
            self.log_error(
                f"[HandledError] determining branch type: {e}",
                "determine_branch_type",
                block.start,
                e,
            )
            return "UNKNOWN"

    def determine_block_type(self, block) -> str:
        """Determine the type of a basic block."""
        try:
            # Check if it's a thunk function (usually just a jump or call)
            if len(block.disassembly_text) <= 2 and any(
                "jmp" in line.tokens[0].text.lower() for line in block.disassembly_text
            ):
                return "THUNK"

            # Check if it contains only data (no valid instructions)
            if all(not line.tokens for line in block.disassembly_text):
                return "DATA"

            # Check if it's padding (usually nops or alignment bytes)
            if all(
                "nop" in line.tokens[0].text.lower() for line in block.disassembly_text
            ):
                return "PADDING"

            # Default to code
            return "CODE"

        except Exception as e:
            self.log_error(
                f"[HandledError] determining block type: {e}",
                "determine_block_type",
                block.start,
                e,
            )
            return "UNKNOWN"

    def extract_constant_references(self, block):
        """Extract constant references"""
        constants = set()
        try:
            for line in block.disassembly_text:
                for token in line.tokens:
                    # Immediate values
                    if token.type == InstructionTextTokenType.IntegerToken:
                        constants.add(f"IMM:{token.text}")

                    # Possible addresses (data references)
                    elif token.type == InstructionTextTokenType.PossibleAddressToken:
                        constants.add(f"DATA:{token.text}")

                    # Check for memory references [offset]
                    elif "[" in token.text and "]" in token.text:
                        offset = token.text.split("[")[1].split("]")[0].strip()
                        if offset.startswith("0x") or offset.isdigit():
                            constants.add(f"OFF:{offset}")
                            constants.add(f"IMM:[{offset}]")
        except Exception as e:
            self.log_error(
                f"[HandledError] extracting constant references: {e}",
                "extract_constant_references",
                block.start,
                e,
            )

        return list(constants)

    @staticmethod
    def run_from_command_line():
        """Run the decompiler as a standalone script from the command line."""
        import argparse
        import json

        # Add a custom JSON encoder to handle Binary Ninja types
        class BinaryNinjaEncoder(json.JSONEncoder):
            def default(self, obj):
                # Handle Binary Ninja specific types
                if hasattr(obj, "value") and hasattr(
                    obj, "confidence"
                ):  # OffsetWithConfidence, etc.
                    return obj.value
                if hasattr(obj, "__str__"):  # Fallback for other objects
                    return str(obj)
                return json.JSONEncoder.default(self, obj)

        parser = argparse.ArgumentParser(description="Binary Ninja Decompiler")
        parser.add_argument("filepath", help="Path to the binary file to analyze")
        parser.add_argument(
            "--output", "-o", help="Output JSON file path (default: stdout)"
        )
        parser.add_argument(
            "--timeout",
            "-t",
            type=int,
            default=1200,
            help="Analysis timeout in seconds (default: 1200)",
        )
        args = parser.parse_args()

        set_license(binaryninja)

        # Setup logging

        logging.basicConfig(
            level=logging.INFO,
            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
        )
        logger = logging.getLogger("BinjaDecompiler")

        # Run decompiler
        with BinjaDecompiler(args.filepath, args.timeout, logger) as decompiler:
            success = decompiler.extract()

            if not success:
                logger.error("Analysis failed")
                return 1

            # Output results
            if args.output:
                with open(args.output, "w") as f:
                    json.dump(decompiler.analysis_results, f, cls=BinaryNinjaEncoder)
                logger.info(f"Results written to {args.output}")
            else:
                print(json.dumps(decompiler.analysis_results, cls=BinaryNinjaEncoder))

        return 0


# Run as script if executed directly
if __name__ == "__main__":
    import sys

    sys.exit(BinjaDecompiler.run_from_command_line())