Chang Lu

import ghidra.app.script.GhidraScript;
import ghidra.program.model.listing.*;
import ghidra.app.decompiler.*;
import ghidra.program.model.address.*;
import org.json.JSONObject;
import org.json.JSONArray;
import java.security.MessageDigest;
import java.nio.charset.StandardCharsets;
import ghidra.program.model.lang.OperandType;
import java.util.*;
import ghidra.program.model.block.*;
import ghidra.util.exception.CancelledException;
import ghidra.program.model.symbol.Reference;
import ghidra.program.model.symbol.Symbol;
import ghidra.program.model.symbol.SymbolTable;
import java.util.Arrays;
import com.sangupta.murmur.Murmur3;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.Executors;

public class GhidraDecompilerScript extends GhidraScript {
    private Listing listing;
    private BasicBlockModel basicBlockModel;
    private JSONArray errors;
    private DecompInterface decompInterface;  

    private enum InstructionType {
        // Data Movement
        GENERAL_DATA_MOVEMENT,    // mov, lea, xchg
        STACK_MANAGEMENT,         // push, pop, enter, leave
        STRING_MANIPULATION,      // movs, lods, stos, cmps
        
        // Arithmetic
        BASIC_ARITHMETIC,         // add, sub, inc, dec
        MULTIPLICATION_DIVISION,  // mul, div, imul, idiv
        CARRY_ARITHMETIC,         // adc, sbb
        
        // Logical
        BITWISE_LOGIC,           // and, or, xor, not
        CONDITIONAL_LOGIC,        // test, cmp, setX
        
        // Control Flow
        UNCONDITIONAL_JUMP,      // jmp
        CONDITIONAL_JUMP,        // je, jne, jl, jg, etc
        FUNCTION_CONTROL,        // call, ret
        LOOPING,                 // loop, loopz, loopnz
        
        // System
        SYSTEM_CALLS,            // syscall, int, sysenter
        PRIVILEGED_INSTRUCTIONS, // hlt, cli, sti
        CPU_FEATURES,            // cpuid, rdtsc
        
        // SIMD & FPU
        SSE_SIMD,               // SSE instructions
        AVX_SIMD,               // AVX instructions
        BASIC_FPU,              // fld, fst, fstp
        FPU_ARITHMETIC,         // fadd, fsub, etc
        
        // Bit Operations
        SHIFT_ROTATE,           // shl, shr, rol, ror
        BIT_TEST_MODIFY,        // bt, bts, btr, btc
        
        // Special
        CRYPTOGRAPHIC_OPS,      // aesenc, aesdec, sha1rnds4
        MISC_OPS               // nop, ud2, etc
    }

    private enum BranchType {
        DIRECT, CONDITIONAL, CALL, RETURN, FALLTHROUGH, UNKNOWN
    }

    private enum ApiCategory {
        FILE_OP(Arrays.asList("CreateFile", "ReadFile", "WriteFile", "DeleteFile", "SetFilePointer", "CopyFile", "MoveFile", "FindFirstFile", "FindNextFile")),
        MEMORY_OP(Arrays.asList("VirtualAlloc", "VirtualFree", "HeapAlloc", "HeapFree", "LocalAlloc", "GlobalAlloc", "MapViewOfFile", "VirtualProtect")),
        NETWORK_OP(Arrays.asList("socket", "connect", "bind", "send", "recv", "WSAStartup", "InternetOpen", "InternetConnect", "HttpOpenRequest", "HttpSendRequest", "InternetReadFile", "URLDownloadToFile")),
        REGISTRY_OP(Arrays.asList("RegOpenKey", "RegCreateKey", "RegSetValue", "RegQueryValue", "RegDeleteKey", "RegEnumKey", "RegFlushKey")),
        PROCESS_OP(Arrays.asList("CreateProcess", "OpenProcess", "TerminateProcess", "GetProcessId", "CreateProcessAsUser", "NtCreateProcess")),
        THREAD_OP(Arrays.asList("CreateThread", "SuspendThread", "ResumeThread", "CreateRemoteThread", "SetThreadContext", "GetThreadContext")),
        INJECTION_OP(Arrays.asList("WriteProcessMemory", "VirtualAllocEx", "NtWriteVirtualMemory", "SetWindowsHookEx", "QueueUserAPC", "NtMapViewOfSection")),
        EVASION_OP(Arrays.asList("IsDebuggerPresent", "CheckRemoteDebuggerPresent", "NtQueryInformationProcess", "GetTickCount", "OutputDebugString", "Sleep", "QueryPerformanceCounter")),
        SPYING_OP(Arrays.asList("GetAsyncKeyState", "GetKeyboardState", "GetKeyState", "GetForegroundWindow", "SetWindowsHookEx", "BitBlt", "GetClipboardData")),
        SYSTEM_OP(Arrays.asList("CreateToolhelp32Snapshot", "EnumDeviceDrivers", "EnumProcesses", "GetSystemDirectoryA", "GetLogicalDrives")),
        SERVICE_OP(Arrays.asList("CreateServiceA", "OpenServiceA", "StartServiceA", "DeleteService", "OpenSCManagerA", "ControlService")),
        CRYPTO_OP(Arrays.asList("CryptAcquireContext", "CryptGenKey", "CryptEncrypt", "CryptDecrypt", "CryptCreateHash", "CryptHashData", "CryptGenRandom")),
        DLL_OP(Arrays.asList("LoadLibrary", "GetProcAddress", "FreeLibrary", "LdrLoadDll")),
        UNKNOWN_OP(Collections.emptyList());

        private final List<String> apis;

        ApiCategory(List<String> apis) {
            this.apis = apis;
        }

        public static ApiCategory fromApi(String apiName) {
            for (ApiCategory category : values()) {
                if (category.apis.stream().anyMatch(api -> apiName.startsWith(api))) {
                    return category;
                }
            }
            return UNKNOWN_OP;
        }
    }

    private static final String[] COMMON_OPCODES = {
        // Core instructions (tracked individually)
        "MOV", "PUSH", "POP", "LEA", "CALL", "RET",         // Data movement and control
        "ADD", "SUB", "MUL", "DIV",                         // Basic arithmetic
        "AND", "OR", "XOR", "NOT",                          // Logical operations
        "JMP", "JE", "JNE",                                 // Basic jumps
        "TEST", "CMP",                                      // Comparisons
        
        // Grouped categories (aggregated tracking)
        "SIMD_MOVE",      // MOVAPS, MOVDQA, MOVDQU, etc.
        "COND_JUMP_EXT",  // Other conditional jumps (JG, JL, JGE, etc.)
        "STRING_OP",      // MOVS, STOS, LODS, SCAS, CMPS
        "STACK_ADV",      // ENTER, LEAVE, PUSHA, POPA
        "ARITHMETIC_ADV", // IMUL, IDIV, ADC, SBB
        "BIT_OP",        // SHL, SHR, SAR, ROL, ROR, etc.
        "FPU_OP",        // FLD, FST, FADD, etc.
        "SYSTEM_OP",     // SYSCALL, INT, SYSENTER
        "CRYPTO_OP",     // AES*, SHA* instructions
        "MISC_OP"        // Rare but interesting (CPUID, RDTSC, etc.)
    };
    private Map<String, Integer> opcodeIndex;
    private Map<String, String> opcodeCategories;

    private void initializeOpcodeMaps() {
        opcodeIndex = new HashMap<>();
        opcodeCategories = new HashMap<>();
        
        for (int i = 0; i < COMMON_OPCODES.length; i++) {
            opcodeIndex.put(COMMON_OPCODES[i], i);
            
            // Categorize opcodes
            String opcode = COMMON_OPCODES[i];
            
            // String Operations (checking these first to avoid MOV confusion)
            if (opcode.matches("MOVS.*|STOS.*|LODS.*|SCAS.*|CMPS.*|REP.*") && 
                !opcode.startsWith("MOVSX") && !opcode.startsWith("MOVZX")) {
                opcodeCategories.put(opcode, "STRING_MANIPULATION");
            }
            
            // Data Movement (after string ops to avoid MOVS confusion)
            else if ((opcode.startsWith("MOV")) || 
                     opcode.equals("LEA") || opcode.equals("XCHG")) {
                opcodeCategories.put(opcode, "DATA_MOVEMENT");
            }
            
            // Stack Operations
            else if (opcode.matches("PUSH|POP|ENTER|LEAVE|PUSHA|POPA")) {
                opcodeCategories.put(opcode, "STACK_MANAGEMENT");
            }
            
            // Control Flow (non-conditional)
            else if (opcode.matches("JMP|CALL|RET")) {
                opcodeCategories.put(opcode, "CONTROL_FLOW");
            }
            
            // Conditional Jumps and Loops
            else if (opcode.startsWith("J") || opcode.startsWith("LOOP")) {
                opcodeCategories.put(opcode, "CONDITIONAL_JUMP");
            }
            
            // Arithmetic
            else if (opcode.matches("ADD|SUB|MUL|DIV|I?MUL|I?DIV|ADC|SBB|INC|DEC|NEG")) {
                opcodeCategories.put(opcode, "ARITHMETIC");
            }
            
            // Logical
            else if (opcode.matches("AND|OR|XOR|NOT|TEST|CMP")) {
                opcodeCategories.put(opcode, "LOGICAL");
            }
            
            // Shifts & Rotates
            else if (opcode.matches("SHL|SHR|SAR|SAL|ROL|ROR|RCL|RCR")) {
                opcodeCategories.put(opcode, "SHIFT_ROTATE");
            }
            
            // System & Interrupts
            else if (opcode.matches("SYSCALL|INT.*|SYSENTER|SYSEXIT|SGDT|SIDT|SLDT|WRMSR|RDMSR")) {
                opcodeCategories.put(opcode, "SYSTEM_CALLS");
            }
            
            // Floating Point
            else if (opcode.matches("F.*")) {
                opcodeCategories.put(opcode, "FPU_ARITHMETIC");
            }
            
            // System Information and Random Number Generation
            else if (opcode.matches("PUSHF|POPF|CPUID|RDTSC|RDRAND|RDSEED")) {
                opcodeCategories.put(opcode, "CPU_FEATURES");
            }
            
            // Cryptography
            else if (opcode.startsWith("AES") || opcode.startsWith("SHA")) {
                opcodeCategories.put(opcode, "CRYPTOGRAPHIC");
            }
            
            // Miscellaneous (including flag operations)
            else {
                opcodeCategories.put(opcode, "MISC");
            }
        }
    }

    private static final int MIN_FUNCTION_SIZE = 10;  // instructions
    private static final int MIN_BLOCK_SIZE = 4;      // instructions
    private static final int INVALID_STACK_SIZE = -1;

    private void logError(String message, String functionName, String address, Exception e, String errorLocation) {
        String errorMsg = String.format("Error in function %s at %s: %s%s", 
            functionName, 
            address, 
            message,
            e != null ? " - " + e.getMessage() : ""  // Only add exception message if e is not null
        );
        println(errorMsg);
        
        // Add to errors array
        JSONObject error = new JSONObject();
        error.put("function_name", functionName);
        error.put("function_address", address);
        error.put("error_location", errorLocation);
        error.put("error_message", message);
        error.put("error_details", e != null ? e.getMessage() : "");
        error.put("error_type", e != null ? e.getClass().getSimpleName() : "Unknown");
        error.put("timestamp", System.currentTimeMillis());
        errors.put(error);
    }

    private String calculateSha256(String input) throws Exception {
        MessageDigest digest = MessageDigest.getInstance("SHA-256");
        byte[] hash = digest.digest(input.getBytes(StandardCharsets.UTF_8));
        StringBuilder hexString = new StringBuilder();
        for (byte b : hash) {
            String hex = Integer.toHexString(0xff & b);
            if (hex.length() == 1) hexString.append('0');
            hexString.append(hex);
        }
        return hexString.toString();
    }

    @Override
    public void run() throws Exception {
        ExecutorService mainExecutor = null;
        final int FUNCTION_TIMEOUT_SECONDS = 90;  // 1.5 minutes per function
        final int GLOBAL_TIMEOUT_SECONDS = 1080 ;    // 18 minutes global, 20 is the total timeout from Python

        try {
            // Initialize listing
            listing = currentProgram.getListing();
            initializeOpcodeMaps();
            
            // // Validate input arguments (expecting SHA256 of binary)
            String[] args = getScriptArgs();
            if (args.length < 1) {
                println("Error: SHA256 argument is required");
                return;
            }
            String binarySha256 = args[0];
            
            // Initialize decompiler interface
            decompInterface = new DecompInterface();
            DecompileOptions options = new DecompileOptions();
            // DecompInterface decompInterface = new DecompInterface();
            decompInterface.setOptions(options);
            
            // Open the current program
            if (!decompInterface.openProgram(currentProgram)) {
                println("Error: Could not open program for decompilation");
                return;
            }
            
            basicBlockModel = new BasicBlockModel(currentProgram);

            // Create main executor for the whole analysis
            mainExecutor = Executors.newSingleThreadExecutor();

            Future<?> analysisTask = mainExecutor.submit(() -> {
                try {

                    // Create main JSON object
                    JSONObject outputJson = new JSONObject();
                    outputJson.put("sha256", binarySha256);
                    JSONArray decompiled = new JSONArray();
                    outputJson.put("decompiled", decompiled);
                    JSONArray disassembled = new JSONArray();
                    outputJson.put("disassembled", disassembled);
                    JSONArray cfg = new JSONArray();
                    outputJson.put("cfg", cfg);

                    // Initialize the errors array
                    errors = new JSONArray();
                    outputJson.put("errors", errors);

                    // Iterate through all functions
                    FunctionIterator functions = currentProgram.getFunctionManager().getFunctions(true);
                    for (Function function : functions) {
                        try {
                            // Create executor service for timeouts
                            ExecutorService executor = Executors.newSingleThreadExecutor();
                            // final int TIMEOUT_SECONDS = 60;  // 1 minute timeout

                            // 1. Decompile block with timeout
                            Future<?> decompileFuture = executor.submit(() -> {
                                try {
                                    DecompileResults results = decompInterface.decompileFunction(function, 30, monitor);
                                    if (results.decompileCompleted()) {
                                        String decompiledCode = results.getDecompiledFunction().getC();
                                        JSONObject functionJson = new JSONObject();
                                        String contentHash = calculateSha256(decompiledCode);
                                        functionJson.put("decompiled_content_hash", contentHash);
                                        functionJson.put("decompiled_function", decompiledCode);
                                        functionJson.put("decompiled_function_name", function.getName());
                                        functionJson.put("decompiled_function_address", function.getEntryPoint().toString());
                                        functionJson.put("function_type", determineFunctionType(function));
                                        decompiled.put(functionJson);
                                    }
                                } catch (Exception e) {
                                    logError("Failed in decompilation block", function.getName(), 
                                            function.getEntryPoint().toString(), e, "decompile");
                                }
                            });

                            try {
                                decompileFuture.get(FUNCTION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                            } catch (TimeoutException e) {
                                logError("Timeout in decompilation block", function.getName(), 
                                        function.getEntryPoint().toString(), null, "decompile");
                                decompileFuture.cancel(true);
                            }

                            // 2. Disassembly block with timeout
                            Future<?> disassemblyFuture = executor.submit(() -> {
                                try {
                                    JSONObject disassemblyJson = extractDisassembly(function);
                                    if (disassemblyJson != null) {
                                        disassemblyJson.put("function_type", determineFunctionType(function));
                                        disassembled.put(disassemblyJson);
                                    }
                                } catch (Exception e) {
                                    logError("Failed in disassembly block", function.getName(), 
                                            function.getEntryPoint().toString(), e, "disassembly");
                                }
                            });

                            try {
                                disassemblyFuture.get(FUNCTION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                            } catch (TimeoutException e) {
                                logError("Timeout in disassembly block", function.getName(), 
                                        function.getEntryPoint().toString(), null, "disassembly");
                                disassemblyFuture.cancel(true);
                            }

                            // 3. CFG block with timeout
                            Future<?> cfgFuture = executor.submit(() -> {
                                try {
                                    CodeBlockIterator blocks = basicBlockModel.getCodeBlocksContaining(function.getBody(), monitor);
                                    while (blocks.hasNext()) {
                                        CodeBlock block = blocks.next();
                                        JSONObject blockJson = extractBasicBlock(block, function);
                                        if (blockJson != null) {
                                            cfg.put(blockJson);
                                        }
                                    }
                                } catch (Exception e) {
                                    logError("Failed in CFG block", function.getName(), 
                                            function.getEntryPoint().toString(), e, "cfg");
                                }
                            });

                            try {
                                cfgFuture.get(FUNCTION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                            } catch (TimeoutException e) {
                                logError("Timeout in CFG block", function.getName(), 
                                        function.getEntryPoint().toString(), null, "cfg");
                                cfgFuture.cancel(true);
                            }

                            // Shutdown executor
                            executor.shutdownNow();

                        } catch (Exception e) {
                            logError("Failed to process function", function.getName(), 
                                    function.getEntryPoint().toString(), e, "unknown");
                            // Continue with next function
                        }
                    }
                    
                    try {
                        // Print the final JSON to stdout for Python to capture
                        System.out.println(outputJson.toString());
                    } catch (Exception e) {
                        System.out.println("Error: Failed to output final JSON: " + e.getMessage()); 
                    }
                } catch (Exception e) {
                        System.out.println("Fatal error in analysis: " + e.getMessage());
                }
            });

            // Wait for completion with a global timeout
            try {
                analysisTask.get(GLOBAL_TIMEOUT_SECONDS, TimeUnit.SECONDS); // 5 minute global timeout
            } catch (TimeoutException e) {
                println("Global analysis timeout reached");
                Thread.sleep(5000);  // 5 second grace period
            }
        } finally {
            // Cleanup section
            println("Starting cleanup process...");
            if (mainExecutor != null) {
                mainExecutor.shutdownNow(); // Force shutdown of all tasks
                try {
                    if (!mainExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
                        println("Warning: Some tasks did not terminate");
                    }
                } catch (InterruptedException ie) {
                    Thread.currentThread().interrupt();
                }
            }

            cleanup();

            // // Print thread state for debugging
            // println("Active threads after cleanup:");
            // Thread.getAllStackTraces().forEach((thread, stackTrace) -> {
            //     println("Thread: " + thread.getName() + " - State: " + thread.getState());
            // });
            
            // Force exit if we're still hanging
            println("Analysis complete, forcing exit...");
            System.exit(0);  // This is a bit aggressive but will ensure termination
        }
    }

    private void cleanup() {
        println("Executing cleanup...");
        try {
            // Force cleanup of any remaining resources
            if (decompInterface != null) {
                println("Closing decompiler interface...");
                decompInterface.closeProgram();
                decompInterface.dispose();
            }
            
            // Clear any thread locals or static resources
            decompInterface = null;
            listing = null;
            basicBlockModel = null;
            errors = null;
            
            // Request garbage collection
            // System.gc();
            
            println("Cleanup completed successfully");
        } catch (Exception e) {
            println("Error during cleanup: " + e.getMessage());
        }
    }

    private JSONObject extractDisassembly(Function function) throws Exception {
        if (function.isExternal() || 
            function.isThunk() || 
            function.getSymbol().isExternal() ||
            function.getEntryPoint().toString().startsWith("00105")) {  // Your external functions all start with 00105
            return null;
        }
        // Add debug for non-external functions that fail
        if (!function.isExternal() && listing.getInstructionAt(function.getEntryPoint()) == null) {
            println(String.format("Debug: Function %s at %s has no instructions but is not external", 
                function.getName(), function.getEntryPoint().toString()));
            return null;
        }

        JSONObject disassemblyJson = new JSONObject();
        try {
            // Check minimum function size first
            int instructionCount = 0;
            AddressSetView functionBody = function.getBody();
            Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                instructionCount++;
                instruction = instruction.getNext();
            }
            
            if (instructionCount < MIN_FUNCTION_SIZE) {
                return null;  // Skip functions that are too small
            }

            StringBuilder disassemblyBuilder = new StringBuilder();
            StringBuilder[] normalizedBuilders = new StringBuilder[3];  // One for each normalization level
            for (int i = 0; i < 3; i++) {
                normalizedBuilders[i] = new StringBuilder();
            }
            // int instructionCount = 0;

            // Get function body
            instruction = null;
            // AddressSetView functionBody = function.getBody();
            // Instruction instruction = null;

            // Iterate through instructions
            try {
                instruction = listing.getInstructionAt(function.getEntryPoint());
            } catch (Exception e) {
                logError("Failed to get first instruction", function.getName(), 
                        function.getEntryPoint().toString(), e, "extractDisassembly");
                return null;
            }

            if (instruction == null) {
                // Don't log error for external functions
                if (!function.isExternal()) {
                    logError("Failed to get first instruction", function.getName(),
                            function.getEntryPoint().toString(), null, "extractDisassembly");
                }
                return null;
            }

            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                try {
                    // Original disassembly with addresses
                    disassemblyBuilder.append(instruction.getAddress().toString())
                                    .append(": ")
                                    .append(instruction.toString())
                                    .append("\n");
                    
                    // Normalized disassembly (just mnemonics and generic operands)
                    try {
                        String[] normalizedVersions = normalizeInstructionAllLevels(instruction);
                        for (int i = 0; i < 3; i++) {
                            normalizedBuilders[i].append(normalizedVersions[i]);
                            if (isControlFlowInstruction(instruction)) {
                                normalizedBuilders[i].append(" <TARGET>");
                            }
                            normalizedBuilders[i].append("\n");
                        }
                        // normalizedBuilder.append(normalizeInstruction(instruction));
                        // if (isControlFlowInstruction(instruction)) {
                        //     normalizedBuilder.append(" <TARGET>");  // Abstract away specific targets
                        // }
                        // normalizedBuilder.append("\n");
                    } catch (Exception e) {
                        logError("Failed to normalize instruction", function.getName(), 
                                instruction.getAddress().toString(), e, "extractDisassembly");
                        // Continue with next instruction
                    }

                    // instructionCount++;
                    instruction = instruction.getNext();
                } catch (Exception e) {
                    logError("Failed processing instruction", function.getName(), 
                            instruction.getAddress().toString(), e, "extractDisassembly");
                    break; // Exit loop if we can't process further instructions
                }
            }

            String disassemblyStr = disassemblyBuilder.toString();
            try {
                disassemblyJson.put("disassembled_content_hash", calculateSha256(disassemblyStr));
                disassemblyJson.put("fully_normalized_content_hash", calculateSha256(normalizedBuilders[0].toString()));
                disassemblyJson.put("api_normalized_content_hash", calculateSha256(normalizedBuilders[1].toString()));
                disassemblyJson.put("category_normalized_content_hash", calculateSha256(normalizedBuilders[2].toString()));
                disassemblyJson.put("disassembled_function", disassemblyStr);
                disassemblyJson.put("fully_normalized_disassembly", normalizedBuilders[0].toString());
                disassemblyJson.put("api_normalized_disassembly", normalizedBuilders[1].toString());
                disassemblyJson.put("category_normalized_disassembly", normalizedBuilders[2].toString());
                disassemblyJson.put("disassembled_function_name", function.getName());
                disassemblyJson.put("disassembled_function_address", function.getEntryPoint().toString());
                disassemblyJson.put("instruction_count", instructionCount);
                Map<String, Integer> typeFrequencies = collectInstructionTypes(function);
                disassemblyJson.put("instruction_types", new JSONArray(typeFrequencies.keySet()));

                disassemblyJson.put("control_flow_count", countControlFlowInstructions(function));
                disassemblyJson.put("memory_access_pattern", new JSONArray(collectMemoryPatterns(function)));
                disassemblyJson.put("register_usage", new JSONArray(collectRegisterUsage(function)));
                disassemblyJson.put("data_references_count", countDataReferences(function));
                // disassemblyJson.put("opcode_frequency_vector", new JSONArray(computeOpcodeFrequency(function)));
                // disassemblyJson.put("api_calls_vector", new JSONArray(computeApiCallsVector(function)));
                // disassemblyJson.put("minhash_signature", new JSONArray(computeMinHash(function, 16)));  // 16 hashes
                disassemblyJson.put("max_block_size", computeMaxBlockSize(function));
                disassemblyJson.put("num_calls", computeNumCalls(function));
                disassemblyJson.put("stack_size", estimateStackSize(function));
                // Map<String, Double> ratios = calculateInstructionTypeRatios(typeFrequencies);
                // disassemblyJson.put("instruction_type_ratios", new JSONArray(convertRatiosToArray(ratios)));

                // String picHash = calculatePicHash(function);
                // if (picHash != null) {
                //     disassemblyJson.put("pic_hash", picHash);
                // }
  
            } catch (Exception e) {
                logError("Failed to create JSON object", function.getName(), 
                        function.getEntryPoint().toString(), e, "extractDisassembly");
                return null;
            }
            
            return disassemblyJson;
        } catch (Exception e) {
            logError("Fatal error in disassembly extraction", function.getName(), 
                    function.getEntryPoint().toString(), e, "extractDisassembly");
            return null;
        }
    }


    private String[] normalizeInstructionAllLevels(Instruction instruction) {
        StringBuilder[] normalized = new StringBuilder[3];
        for (int i = 0; i < 3; i++) {
            normalized[i] = new StringBuilder();
        }

        try {
            String mnemonic = instruction.getMnemonicString();
            for (int i = 0; i < 3; i++) {
                normalized[i].append(mnemonic);
            }

            for (int i = 0; i < instruction.getNumOperands(); i++) {
                String operand = instruction.getDefaultOperandRepresentation(i);
                String[] normalizedOps = normalizeOperandAllLevels(operand, instruction, i);
                
                for (int level = 0; level < 3; level++) {
                    normalized[level].append(" ").append(normalizedOps[level]);
                }
            }
        } catch (Exception e) {
            // If normalization fails, return simple mnemonic
            String fallback = instruction.getMnemonicString() + " op";
            return new String[]{fallback, fallback, fallback};
        }

        return new String[]{
            normalized[0].toString(),
            normalized[1].toString(),
            normalized[2].toString()
        };
    }

    private String[] normalizeOperandAllLevels(String operand, Instruction instruction, int opIndex) {
        // Returns [fullyNormalized, apiNormalized, categoryNormalized]
        String[] normalizations = new String[3];
        
        operand = operand.trim().toUpperCase();
        
        // Handle API calls
        if (instruction.getOperandType(opIndex) == OperandType.ADDRESS || 
            (operand.startsWith("0X") && isLikelyApi(operand))) {
            
            String apiName = resolveApiName(operand, instruction);
            if (apiName != null) {
                normalizations[0] = "DATA_REF";  // Most abstract
                normalizations[1] = "API_" + apiName;  // Preserve exact API
                normalizations[2] = "API_" + ApiCategory.fromApi(apiName).name();  // Use category
                return normalizations;
            }
        }

        // Handle immediate values
        if (instruction.getOperandType(opIndex) == OperandType.SCALAR) {
            // Level 0: Most abstract
            normalizations[0] = "CONST";
            
            // Level 1: Keep small constants, normalize large ones
            try {
                long value = Long.decode(operand);
                if (value >= -16 && value <= 16) {
                    normalizations[1] = "CONST_" + value;
                } else {
                    normalizations[1] = "CONST_LARGE";
                }
            } catch (NumberFormatException e) {
                normalizations[1] = "CONST";
            }
            
            // Level 2: Distinguish between different types of constants
            if (operand.startsWith("0X")) {
                normalizations[2] = "CONST_HEX";
            } else {
                normalizations[2] = "CONST_DEC";
            }
            return normalizations;
        }
        
        // Handle registers
        if (instruction.getOperandType(opIndex) == OperandType.REGISTER) {
            // Level 0: Most abstract - just register type
            normalizations[0] = normalizeRegister(operand);
            
            // Level 1: Preserve register class but normalize within class
            if (operand.matches("E?[ABCD]X|R\\d+")) {
                normalizations[1] = "GPR_DATA";
            } else if (operand.matches("E?SI|E?DI")) {
                normalizations[1] = "GPR_INDEX";
            } else if (operand.matches("E?[BS]P|R?SP")) {
                normalizations[1] = "GPR_STACK";
            } else if (operand.matches("XMM\\d+")) {
                normalizations[1] = "XMM_REG";
            } else if (operand.matches("ST\\d+")) {
                normalizations[1] = "FPU_REG";
            } else {
                normalizations[1] = "OTHER_REG";
            }
            
            // Level 2: More specific register categorization
            if (operand.startsWith("R")) {
                normalizations[2] = "REG_64";
            } else if (operand.startsWith("E")) {
                normalizations[2] = "REG_32";
            } else if (operand.matches("[ABCD][XHL]|SI|DI|SP|BP")) {
                normalizations[2] = "REG_16_8";
            } else {
                normalizations[2] = "REG_SPECIAL";
            }
            return normalizations;
        }
        
        // Handle memory references
        if (operand.contains("[")) {
            // Level 0: Most abstract
            normalizations[0] = operand.matches(".*\\[.*[+-].*\\]") ? "MEM_OFF" : "MEM";
            
            // Level 1: Distinguish base register types
            String memContent = operand.replaceAll(".*\\[(.*?)\\].*", "$1");
            if (memContent.contains("ESP") || memContent.contains("EBP")) {
                normalizations[1] = "MEM_STACK";
            } else if (memContent.contains("ESI") || memContent.contains("EDI")) {
                normalizations[1] = "MEM_STRING";
            } else {
                normalizations[1] = "MEM_GENERAL";
            }
            
            // Level 2: More detailed memory access pattern
            if (memContent.contains("+") && memContent.contains("*")) {
                normalizations[2] = "MEM_SCALED_INDEX";
            } else if (memContent.contains("+") || memContent.contains("-")) {
                normalizations[2] = "MEM_BASE_OFFSET";
            } else {
                normalizations[2] = "MEM_DIRECT";
            }
            return normalizations;
        }
        
        // Handle string/data references that aren't APIs
        if (instruction.getOperandType(opIndex) == OperandType.ADDRESS || 
            operand.startsWith("0X") || 
            operand.matches(".*_.*")) {
            normalizations[0] = "DATA_REF";
            normalizations[1] = "DATA_" + (operand.startsWith("0X") ? "ADDR" : "SYM");
            normalizations[2] = "DATA_" + (operand.contains("str") || operand.contains("STR") ? "STRING" : "OTHER");
            return normalizations;
        }
        
        // Default case - keep original for all levels if we can't categorize
        normalizations[0] = normalizations[1] = normalizations[2] = operand;
        return normalizations;
    }

    private boolean isLikelyApi(String operand) {
        // Check for common API function patterns
        return operand.matches(".*_[A-Za-z].*") ||  // Has underscore followed by letters
            operand.matches("^[A-Z][a-z]+[A-Z].*") || // PascalCase pattern typical of Win32 APIs
            operand.matches(".*@.*") ||  // Has @ symbol (common in decorated names)
            operand.matches("^_.*[A-Z].*"); // Starts with underscore and has uppercase
    }

    // Helper method to resolve API names
    private String resolveApiName(String operand, Instruction instruction) {
        // Try to get symbol name if available
        Reference[] refs = instruction.getReferencesFrom();
        if (refs != null && refs.length > 0) {
            for (Reference ref : refs) {
                Address toAddr = ref.getToAddress();
                SymbolTable symbolTable = currentProgram.getSymbolTable();
                Symbol[] symbols = symbolTable.getSymbols(toAddr);
                if (symbols != null && symbols.length > 0) {
                    // First try to find a symbol that matches our API patterns
                    for (Symbol sym : symbols) {
                        String name = sym.getName();
                        if (isLikelyApi(name)) {  // reuse your existing API detection
                            name = name.replaceAll("^_*", "")  // Remove leading underscores
                                    .replaceAll("@.*$", "");  // Remove @ decorations
                            return name;
                        }
                    }
                    // If no API-like symbol found, fall back to first symbol
                    String name = symbols[0].getName()
                        .replaceAll("^_*", "")
                        .replaceAll("@.*$", "");
                    return name;
                }
            }
        }
        return null;
    }

    private String normalizeRegister(String register) {
        // Convert to uppercase for consistent matching
        register = register.toUpperCase();
        
        // General purpose registers
        if (register.matches("E?[ABCD]X|E?SI|E?DI|R\\d+")) {
            return "GPR";
        }
        // Stack/base pointers
        if (register.matches("E?[BS]P|R?SP")) {
            return "PTR";
        }
        // SIMD registers
        if (register.matches("XMM\\d+")) {
            return "XMM";
        }
        // FPU registers
        if (register.matches("ST\\d+")) {
            return "FPU";
        }
        // Any other register
        return "REG";
    }

    private boolean isControlFlowInstruction(Instruction instruction) {
        String mnemonic = instruction.getMnemonicString().toUpperCase();
        return mnemonic.startsWith("J") ||
               mnemonic.equals("CALL") ||
               mnemonic.equals("RET") ||
               mnemonic.equals("LOOP");
    }

    private Map<String, Integer> collectInstructionTypes(Function function) {
        Map<String, Integer> typeFrequencies = new HashMap<>();
        try {
            AddressSetView functionBody = function.getBody();
            Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
            
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                try {
                    String mnemonic = instruction.getMnemonicString().toUpperCase();
                    
                    // Data Movement Categories
                    if (mnemonic.matches("MOV|LEA|XCHG")) {
                        incrementFrequency(typeFrequencies, InstructionType.GENERAL_DATA_MOVEMENT.name());
                    }
                    else if (mnemonic.matches("PUSH|POP|ENTER|LEAVE|PUSHA|POPA")) {
                        incrementFrequency(typeFrequencies, InstructionType.STACK_MANAGEMENT.name());
                    }
                    else if (mnemonic.matches("MOVS|LODS|STOS|CMPS|SCAS|REP.*")) {
                        incrementFrequency(typeFrequencies, InstructionType.STRING_MANIPULATION.name());
                    }
                    
                    // Arithmetic Categories
                    else if (mnemonic.matches("ADD|SUB|INC|DEC")) {
                        incrementFrequency(typeFrequencies, InstructionType.BASIC_ARITHMETIC.name());
                    }
                    else if (mnemonic.matches("MUL|DIV|IMUL|IDIV")) {
                        incrementFrequency(typeFrequencies, InstructionType.MULTIPLICATION_DIVISION.name());
                    }
                    else if (mnemonic.matches("ADC|SBB")) {
                        incrementFrequency(typeFrequencies, InstructionType.CARRY_ARITHMETIC.name());
                    }
                    
                    // Logical Categories
                    else if (mnemonic.matches("AND|OR|XOR|NOT")) {
                        incrementFrequency(typeFrequencies, InstructionType.BITWISE_LOGIC.name());
                    }
                    else if (mnemonic.matches("TEST|CMP|SET[A-Z]+")) {
                        incrementFrequency(typeFrequencies, InstructionType.CONDITIONAL_LOGIC.name());
                    }
                    
                    // Control Flow Categories
                    else if (mnemonic.equals("JMP")) {
                        incrementFrequency(typeFrequencies, InstructionType.UNCONDITIONAL_JUMP.name());
                    }
                    else if (mnemonic.matches("J[A-Z]+") && !mnemonic.equals("JMP")) {
                        incrementFrequency(typeFrequencies, InstructionType.CONDITIONAL_JUMP.name());
                    }
                    else if (mnemonic.matches("CALL|RET")) {
                        incrementFrequency(typeFrequencies, InstructionType.FUNCTION_CONTROL.name());
                    }
                    else if (mnemonic.matches("LOOP.*")) {
                        incrementFrequency(typeFrequencies, InstructionType.LOOPING.name());
                    }
                    
                    // System Categories
                    else if (mnemonic.matches("SYSCALL|SYSENTER|INT")) {
                        incrementFrequency(typeFrequencies, InstructionType.SYSTEM_CALLS.name());
                    }
                    else if (mnemonic.matches("HLT|CLI|STI")) {
                        incrementFrequency(typeFrequencies, InstructionType.PRIVILEGED_INSTRUCTIONS.name());
                    }
                    else if (mnemonic.matches("CPUID|RDTSC|RDTSCP")) {
                        incrementFrequency(typeFrequencies, InstructionType.CPU_FEATURES.name());
                    }
                    
                    // SIMD & FPU Categories
                    else if (mnemonic.startsWith("V") && (
                        mnemonic.contains("PS") || mnemonic.contains("PD") || 
                        mnemonic.contains("SS") || mnemonic.contains("SD"))) {
                        incrementFrequency(typeFrequencies, InstructionType.AVX_SIMD.name());
                    }
                    else if (mnemonic.matches(".*(?:PS|PD|SS|SD).*") || 
                            mnemonic.startsWith("MM") || mnemonic.startsWith("P")) {
                        incrementFrequency(typeFrequencies, InstructionType.SSE_SIMD.name());
                    }
                    else if (mnemonic.matches("FLD|FST|FSTP")) {
                        incrementFrequency(typeFrequencies, InstructionType.BASIC_FPU.name());
                    }
                    else if (mnemonic.matches("FADD|FSUB|FMUL|FDIV|FSQRT")) {
                        incrementFrequency(typeFrequencies, InstructionType.FPU_ARITHMETIC.name());
                    }
                    
                    // Bit Operation Categories
                    else if (mnemonic.matches("SHL|SHR|SAR|SAL|ROR|ROL|RCR|RCL")) {
                        incrementFrequency(typeFrequencies, InstructionType.SHIFT_ROTATE.name());
                    }
                    else if (mnemonic.matches("BT|BTS|BTR|BTC|BSF|BSR")) {
                        incrementFrequency(typeFrequencies, InstructionType.BIT_TEST_MODIFY.name());
                    }
                    
                    // Special Categories
                    else if (mnemonic.matches("AES.*|SHA.*|RDRAND|RDSEED")) {
                        incrementFrequency(typeFrequencies, InstructionType.CRYPTOGRAPHIC_OPS.name());
                    }
                    else if (mnemonic.matches("NOP|UD2|INT3")) {
                        incrementFrequency(typeFrequencies, InstructionType.MISC_OPS.name());
                    }
                    
                    instruction = instruction.getNext();
                } catch (Exception e) {
                    logError("Failed to process instruction type", function.getName(), 
                            instruction.getAddress().toString(), e, "collectInstructionTypes");
                }
            }
        } catch (Exception e) {
            logError("Failed to collect instruction types", function.getName(), 
                    function.getEntryPoint().toString(), e, "collectInstructionTypes");
        }
        
        return typeFrequencies;
    }

    private void incrementFrequency(Map<String, Integer> frequencies, String type) {
        frequencies.put(type, frequencies.getOrDefault(type, 0) + 1);
    }

    /**
     * Collect memory access patterns in the function
     */
    private List<String> collectMemoryPatterns(Function function) {
        Set<String> patterns = new HashSet<>();
        try {
            AddressSetView functionBody = function.getBody();
            Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
            
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                try {
                    for (int i = 0; i < instruction.getNumOperands(); i++) {
                        String opRep = instruction.getDefaultOperandRepresentation(i);
                        // Check if operand uses memory (indicated by square brackets)
                        if (opRep.contains("[")) {
                            if (opRep.contains("+") || opRep.contains("-")) {
                                patterns.add("BASE_PLUS_OFFSET");
                            } else if (opRep.contains("*")) {
                                patterns.add("SCALED_INDEX");
                            } else {
                                patterns.add("DIRECT_MEMORY");
                            }
                        }
                    }
                    instruction = instruction.getNext();
                } catch (Exception e) {
                    logError("Failed to process memory pattern", function.getName(), 
                            instruction.getAddress().toString(), e, "collectMemoryPatterns");
                }
            }
        } catch (Exception e) {
            logError("Failed to collect memory patterns", function.getName(), 
                    function.getEntryPoint().toString(), e, "collectMemoryPatterns");
        }
        return new ArrayList<>(patterns);
    }

    /**
     * Collect register usage information
     */
    private List<String> collectRegisterUsage(Function function) {
        Set<String> registers = new HashSet<>();
        try {
            AddressSetView functionBody = function.getBody();
            Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
            
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                try {
                    for (int i = 0; i < instruction.getNumOperands(); i++) {
                        if ((instruction.getOperandType(i) & OperandType.REGISTER) != 0) {
                            String reg = instruction.getDefaultOperandRepresentation(i).toUpperCase();
                            // Categorize register type
                            if (reg.matches("E?[ABCD]X|R\\d+")) registers.add("GPR");
                            else if (reg.matches("E?[BS]P|R?SP")) registers.add("STACK_PTR");
                            else if (reg.matches("E?[SD]I")) registers.add("INDEX_PTR");
                            else if (reg.matches("XMM\\d+")) registers.add("SIMD");
                            else if (reg.matches("ST\\d+")) registers.add("FPU");
                            else registers.add("OTHER");
                        }
                    }
                    instruction = instruction.getNext();
                } catch (Exception e) {
                    logError("Failed to process register usage", function.getName(), 
                            instruction.getAddress().toString(), e, "collectRegisterUsage");
                }
            }
        } catch (Exception e) {
            logError("Failed to collect register usage", function.getName(), 
                    function.getEntryPoint().toString(), e, "collectRegisterUsage");
        }
        return new ArrayList<>(registers);
    }

    /**
     * Count references to data section
     */
    private int countDataReferences(Function function) {
        int count = 0;
        try {
            AddressSetView functionBody = function.getBody();
            Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
            
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                try {
                    // Count operands that access data
                    for (int i = 0; i < instruction.getNumOperands(); i++) {
                        String opRep = instruction.getDefaultOperandRepresentation(i);
                        // Look for data references like symbols, constants
                        if (instruction.getOperandType(i) == OperandType.SCALAR || 
                            opRep.matches(".*_[A-Za-z].*") ||  // Named references
                            (opRep.startsWith("0x") && !opRep.contains("[")) // Direct address without memory indirection
                        ) {
                            count++;
                        }
                    }
                    instruction = instruction.getNext();
                } catch (Exception e) {
                    logError("Failed to process data reference", function.getName(), 
                            instruction.getAddress().toString(), e, "countDataReferences");
                }
            }
        } catch (Exception e) {
            logError("Failed to count data references", function.getName(), 
                    function.getEntryPoint().toString(), e, "countDataReferences");
        }
        return count;
    }

    /**
     * Count control flow instructions (jumps, calls, returns)
     */
    private int countControlFlowInstructions(Function function) {
        int count = 0;
        try {
            AddressSetView functionBody = function.getBody();
            Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
            
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                try {
                    String mnemonic = instruction.getMnemonicString().toUpperCase();
                    // Check for jumps, calls, returns
                    if (mnemonic.startsWith("J") ||     // All jumps
                        mnemonic.equals("CALL") ||      // Calls
                        mnemonic.equals("RET") ||       // Returns
                        mnemonic.equals("LOOP")) {      // Loop instructions
                        count++;
                    }
                    instruction = instruction.getNext();
                } catch (Exception e) {
                    logError("Failed to process control flow instruction", function.getName(), 
                            instruction.getAddress().toString(), e, "countControlFlowInstructions");
                }
            }
        } catch (Exception e) {
            logError("Failed to count control flow instructions", function.getName(), 
                    function.getEntryPoint().toString(), e, "countControlFlowInstructions");
        }
        return count;
    }

    private JSONObject extractBasicBlock(CodeBlock block, Function function) {
        try {
            // Check minimum block size first
            int blockSize = 0;
            AddressIterator addrs = block.getAddresses(true);
            while (addrs.hasNext()) {
                Address addr = addrs.next();
                if (listing.getInstructionAt(addr) != null) {
                    blockSize++;
                }
            }
            
            if (blockSize < MIN_BLOCK_SIZE) {
                return null;  // Skip blocks that are too small
            }

            JSONObject blockJson = new JSONObject();
            String blockInstructions = getBlockInstructions(block);
            
            // Calculate block_id
            String blockIdInput = blockInstructions + 
                                block.getMinAddress().toString() +
                                block.getMaxAddress().toString() +
                                function.getEntryPoint().toString();
            String blockId = calculateSha256(blockIdInput);

            blockJson.put("block_id", blockId);
            blockJson.put("function_address", function.getEntryPoint().toString());
            blockJson.put("block_start_address", block.getMinAddress().toString());
            blockJson.put("block_end_address", block.getMaxAddress().toString());
            blockJson.put("block_instructions", blockInstructions);
            blockJson.put("block_size", blockSize);
            
            // Get predecessor blocks
            JSONArray predecessors = new JSONArray();
            CodeBlockReferenceIterator preds = block.getSources(monitor);
            while (preds.hasNext()) {
                CodeBlockReference ref = preds.next();
                predecessors.put(ref.getSourceAddress().toString());
            }
            if (predecessors.length() > 0) {
                blockJson.put("predecessor_blocks", predecessors);
            }

            // Get successor blocks
            JSONArray successors = new JSONArray();
            CodeBlockReferenceIterator succs = block.getDestinations(monitor);
            while (succs.hasNext()) {
                CodeBlockReference ref = succs.next();
                successors.put(ref.getDestinationAddress().toString());
            }
            if (successors.length() > 0) {
                blockJson.put("successor_blocks", successors);
            }

            // Determine if entry/exit block
            blockJson.put("is_entry_block", block.getMinAddress().equals(function.getEntryPoint()));
            blockJson.put("is_exit_block", successors.length() == 0);

            BranchType branchType = determineBranchType(block);
            blockJson.put("branch_type", branchType.name());

            // Add normalized instructions
            StringBuilder[] normalizedBuilders = new StringBuilder[3];  // One for each level
            for (int i = 0; i < 3; i++) {
                normalizedBuilders[i] = new StringBuilder();
            }

            addrs = block.getAddresses(true);
            while (addrs.hasNext()) {
                Address addr = addrs.next();
                Instruction instr = listing.getInstructionAt(addr);
                if (instr != null) {
                    String[] normalizedVersions = normalizeInstructionAllLevels(instr);
                    for (int i = 0; i < 3; i++) {
                        normalizedBuilders[i].append(normalizedVersions[i]);
                        if (isControlFlowInstruction(instr)) {
                            normalizedBuilders[i].append(" <TARGET>");
                        }
                        normalizedBuilders[i].append("\n");
                    }
                }
            }

            blockJson.put("fully_normalized_instructions", normalizedBuilders[0].toString());
            blockJson.put("api_normalized_instructions", normalizedBuilders[1].toString());
            blockJson.put("category_normalized_instructions", normalizedBuilders[2].toString());

            Set<String> constants = extractConstantReferences(block);
            if (!constants.isEmpty()) {
                blockJson.put("referenced_constants", new JSONArray(constants));
            }
            
            return blockJson;
        } catch (Exception e) {
            logError("Failed to process basic block", function.getName(), 
                    block.getMinAddress().toString(), e, "extractBasicBlock");
            return null;
        }
    }

    private String getBlockInstructions(CodeBlock block) {
        StringBuilder instructions = new StringBuilder();
        AddressIterator addresses = block.getAddresses(true);
        while (addresses.hasNext()) {
            Address addr = addresses.next();
            Instruction instr = listing.getInstructionAt(addr);
            if (instr != null) {
                instructions.append(addr.toString())
                        .append(": ")
                        .append(instr.toString())
                        .append("\n");
            }
        }
        return instructions.toString();
    }

    private BranchType determineBranchType(CodeBlock block) {
        try {
            Address lastInstrAddr = block.getMaxAddress();
            Instruction lastInstr = listing.getInstructionAt(lastInstrAddr);
            if (lastInstr == null) {
                return BranchType.UNKNOWN;
            }

            String mnemonic = lastInstr.getMnemonicString().toUpperCase();
            
            // Check for return instructions
            if (mnemonic.equals("RET") || mnemonic.equals("RETN")) {
                return BranchType.RETURN;
            }
            
            // Check for call instructions
            if (mnemonic.equals("CALL")) {
                return BranchType.CALL;
            }
            
            // Check for jumps
            if (mnemonic.startsWith("J")) {
                // Unconditional jumps
                if (mnemonic.equals("JMP")) {
                    return BranchType.DIRECT;
                }
                // Conditional jumps (JE, JNE, JG, JLE, etc.)
                return BranchType.CONDITIONAL;
            }
            
            // Check number of successors
            CodeBlockReferenceIterator succs = block.getDestinations(monitor);
            if (succs.hasNext()) {
                return BranchType.FALLTHROUGH;
            }
            
            return BranchType.UNKNOWN;
        } catch (Exception e) {
            logError("Failed to determine branch type", "", block.getMinAddress().toString(), e, "determineBranchType");
            return BranchType.UNKNOWN;
        }
    }

    private Set<String> extractConstantReferences(CodeBlock block) {
        Set<String> constants = new HashSet<>();
        try {
            AddressIterator addresses = block.getAddresses(true);
            while (addresses.hasNext()) {
                Address addr = addresses.next();
                Instruction instr = listing.getInstructionAt(addr);
                if (instr == null) continue;

                // Process each operand
                for (int i = 0; i < instr.getNumOperands(); i++) {
                    int opType = instr.getOperandType(i);
                    String opRep = instr.getDefaultOperandRepresentation(i);
                    
                    // Immediate values
                    if ((opType & OperandType.SCALAR) != 0) {
                        constants.add("IMM:" + opRep);
                    }
                    
                    // Memory offsets
                    if (opRep.contains("[")) {
                        String offset = opRep.replaceAll(".*\\[(.*?)\\].*", "$1");
                        if (offset.matches("^[+-]?\\d+$") || offset.startsWith("0x")) {
                            constants.add("OFF:" + offset);
                        }
                    }
                    
                    // Data references
                    Reference[] refs = instr.getOperandReferences(i);
                    if (refs != null) {
                        for (Reference ref : refs) {
                            if (ref.getReferenceType().isData()) {
                                constants.add("DATA:" + ref.getToAddress());
                            }
                        }
                    }
                }
            }
        } catch (Exception e) {
            logError("Failed to extract constants", "", block.getMinAddress().toString(), e, "extractConstantReferences");
        }
        return constants;
    }

    private String determineFunctionType(Function function) {
        try {
            if (function.isExternal()) {
                return "EXTERNAL";
            }
            if (function.isThunk()) {
                return "THUNK";
            }
            // Check if it's a library function by name pattern or location
            String name = function.getName();
            if (name.startsWith("__") || 
                name.contains("@") || 
                function.getEntryPoint().toString().startsWith("00105")) {
                return "LIBRARY";
            }
            return "USER";
        } catch (Exception e) {
            logError("Failed to determine function type", function.getName(), 
                    function.getEntryPoint().toString(), e, "determineFunctionType");
            return "UNKNOWN";
        }
    }

    private float[] computeOpcodeFrequency(Function function) {
        float[] frequencies = new float[COMMON_OPCODES.length];
        int totalInstructions = 0;
        
        try {
            AddressSetView functionBody = function.getBody();
            Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
            
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                String mnemonic = instruction.getMnemonicString();
                String normalized = normalizeOpcode(mnemonic);
                
                Integer index = opcodeIndex.get(normalized);
                if (index != null) {
                    frequencies[index]++;
                }
                
                totalInstructions++;
                instruction = instruction.getNext();
            }
            
            // Normalize
            if (totalInstructions > 0) {
                for (int i = 0; i < frequencies.length; i++) {
                    frequencies[i] /= totalInstructions;
                }
            }
            
        } catch (Exception e) {
            logError("Failed to compute opcode frequencies", function.getName(), 
                function.getEntryPoint().toString(), e, "computeOpcodeFrequency");
        }
        
        return frequencies;
    }


    private String normalizeOpcode(String mnemonic) {
        String normalized = mnemonic.trim().toUpperCase();
        
        // Handle prefixes
        if (normalized.startsWith("REP") || normalized.startsWith("REPE") || normalized.startsWith("REPNE")) {
            normalized = normalized.substring(normalized.indexOf(' ') + 1);
        }
        
        // Direct matches first
        if (opcodeIndex.containsKey(normalized)) {
            return normalized;
        }
        
        // Group categorization
        if (normalized.matches("MOV[AU]PS|MOVDQ[AU]|VMOVDQ[AU]|VMOV[AU]PS")) {
            return "SIMD_MOVE";
        }
        
        if (normalized.matches("J[GLABE][E]?|JG?[EZSC]|JN[GLABE][E]?|JN[EZSC]")) {
            return "COND_JUMP_EXT";
        }
        
        if (normalized.matches("MOVS|STOS|LODS|SCAS|CMPS|MOVSB|MOVSW|MOVSD")) {
            return "STRING_OP";
        }
        
        if (normalized.matches("ENTER|LEAVE|PUSHA|POPA")) {
            return "STACK_ADV";
        }
        
        if (normalized.matches("IMUL|IDIV|ADC|SBB|NEG|BSWAP")) {
            return "ARITHMETIC_ADV";
        }
        
        if (normalized.matches("SHL|SHR|SAR|SAL|ROL|ROR|RCL|RCR|BT[SR]?|BSF|BSR")) {
            return "BIT_OP";
        }
        
        if (normalized.startsWith("F")) {
            return "FPU_OP";
        }
        
        if (normalized.matches("SYSCALL|INT|SYSENTER|SYSEXIT")) {
            return "SYSTEM_OP";
        }
        
        if (normalized.matches("AES.*|SHA.*|RDRAND|RDSEED")) {
            return "CRYPTO_OP";
        }
        
        if (normalized.matches("CPUID|RDTSC|UD2|HLT|PAUSE")) {
            return "MISC_OP";
        }
        
        return normalized;  // Keep as is if no categorization matches
    }


    // method for API calls vector
    private float[] computeApiCallsVector(Function function) {
        // Initialize vector with one position per API category
        float[] apiVector = new float[ApiCategory.values().length];
        int totalCalls = 0;
        
        AddressSetView functionBody = function.getBody();
        Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
        
        while (instruction != null && functionBody.contains(instruction.getAddress())) {
            if (instruction.getMnemonicString().equals("CALL")) {
                for (int i = 0; i < instruction.getNumOperands(); i++) {
                    String apiName = resolveApiName(
                        instruction.getDefaultOperandRepresentation(i), 
                        instruction
                    );
                    if (apiName != null) {
                        ApiCategory category = ApiCategory.fromApi(apiName);
                        apiVector[category.ordinal()]++;
                        totalCalls++;
                    }
                }
            }
            instruction = instruction.getNext();
        }
        
        // Normalize to frequencies
        if (totalCalls > 0) {
            for (int i = 0; i < apiVector.length; i++) {
                apiVector[i] /= totalCalls;
            }
        }
        
        return apiVector;
    }

    private long[] computeMinHash(Function function, int numHashes) {
        long[] signature = new long[numHashes];
        Arrays.fill(signature, Long.MAX_VALUE);
        
        AddressSetView functionBody = function.getBody();
        Instruction instruction = listing.getInstructionAt(function.getEntryPoint());

        while (instruction != null && functionBody.contains(instruction.getAddress())) {
            String[] normalizedVersions = normalizeInstructionAllLevels(instruction);
            String token = normalizedVersions[0];  // Fully normalized instruction
            
            // Hash token using numHashes different hash functions
            for (int i = 0; i < numHashes; i++) {
                long hash = murmurHash64(token, i) & Long.MAX_VALUE;  // Ensure non-negative
                signature[i] = Math.min(signature[i], hash);
            }

            instruction = instruction.getNext();
        }

        return signature;
    }

    // MurmurHash3 function for better MinHashing
    private long murmurHash64(String token, int seed) {
        return Murmur3.hash_x64_128(token.getBytes(StandardCharsets.UTF_8), token.length(), seed)[0];
    }

    private long calculateTokenHash(String token, long seed) {
        // MurmurHash3-like hashing
        long h = seed;
        
        for (char c : token.toCharArray()) {
            h ^= c;
            h *= 0x5bd1e995;
            h ^= h >>> 13;
        }
        
        return h;
    }

    /*
    PicHash calculation which:
    - Uses the fully normalized instruction representation
    - Creates a position-independent hash by normalizing addresses
    - Takes first 64 bits (16 chars) of SHA-256
    */
    private String calculatePicHash(Function function) {
        try {
            StringBuilder normalizedFunction = new StringBuilder();
            AddressSetView functionBody = function.getBody();
            Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
            
            // Check minimum size threshold
            int instructionCount = 0;
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                instructionCount++;
                instruction = instruction.getNext();
            }
            if (instructionCount < MIN_FUNCTION_SIZE) {
                return null;
            }
            
            // Reset to start of function for actual processing
            instruction = listing.getInstructionAt(function.getEntryPoint());
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                String[] normalizedVersions = normalizeInstructionAllLevels(instruction);
                // Use fully normalized version (index 0) for PicHash
                normalizedFunction.append(normalizedVersions[0]).append("\n");
                instruction = instruction.getNext();
            }
            
            // Calculate SHA-256 and take first 16 characters (64 bits)
            return calculateSha256(normalizedFunction.toString()).substring(0, 16);
        } catch (Exception e) {
            logError("Failed to calculate PicHash", function.getName(), 
                    function.getEntryPoint().toString(), e, "calculatePicHash");
            return null;
        }
    }

    private int computeMaxBlockSize(Function function) {
        try {
            int maxSize = 0;
            CodeBlockIterator blocks = basicBlockModel.getCodeBlocksContaining(function.getBody(), monitor);
            while (blocks.hasNext()) {
                CodeBlock block = blocks.next();
                int blockSize = 0;
                AddressIterator addrs = block.getAddresses(true);
                while (addrs.hasNext()) {
                    Address addr = addrs.next();
                    if (listing.getInstructionAt(addr) != null) {
                        blockSize++;
                    }
                }
                maxSize = Math.max(maxSize, blockSize);
            }
            return maxSize;
        } catch (Exception e) {
            logError("Failed to compute max block size", function.getName(), 
                    function.getEntryPoint().toString(), e, "computeMaxBlockSize");
            return 0;
        }
    }

    private int computeNumCalls(Function function) {
        try {
            int callCount = 0;
            AddressSetView functionBody = function.getBody();
            Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
            
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                if (instruction.getMnemonicString().toUpperCase().equals("CALL")) {
                    callCount++;
                }
                instruction = instruction.getNext();
            }
            return callCount;
        } catch (Exception e) {
            logError("Failed to compute call count", function.getName(), 
                    function.getEntryPoint().toString(), e, "computeNumCalls");
            return 0;
        }
    }

    private int estimateStackSize(Function function) {
        try {
            // Look for stack adjustment in function prologue
            Instruction instruction = listing.getInstructionAt(function.getEntryPoint());
            AddressSetView functionBody = function.getBody();
            boolean foundProlog = false;
            int stackSize = INVALID_STACK_SIZE;
            
            // Look for common prolog patterns
            while (instruction != null && functionBody.contains(instruction.getAddress())) {
                String mnemonic = instruction.getMnemonicString().toUpperCase();
                
                // Look for PUSH EBP/RBP followed by MOV EBP/RBP, ESP/RSP
                if (mnemonic.equals("PUSH") && 
                    instruction.getDefaultOperandRepresentation(0).toUpperCase().matches(".*BP")) {
                    foundProlog = true;
                }
                // Look for SUB ESP/RSP, immediate
                else if (foundProlog && mnemonic.equals("SUB") && 
                        instruction.getDefaultOperandRepresentation(0).toUpperCase().matches(".*SP")) {
                    String immValue = instruction.getDefaultOperandRepresentation(1);
                    try {
                        // Handle hex values
                        if (immValue.startsWith("0x")) {
                            stackSize = Integer.parseInt(immValue.substring(2), 16);
                        } else {
                            stackSize = Integer.parseInt(immValue);
                        }
                        break;
                    } catch (NumberFormatException nfe) {
                        // If we can't parse the value, continue
                        continue;
                    }
                }
                instruction = instruction.getNext();
            }
            return stackSize;
        } catch (Exception e) {
            logError("Failed to estimate stack size", function.getName(), 
                    function.getEntryPoint().toString(), e, "estimateStackSize");
            return INVALID_STACK_SIZE;
        }
    }

    // Helper method to calculate instruction type ratios
    private Map<String, Double> calculateInstructionTypeRatios(Map<String, Integer> typeFrequencies) {
        Map<String, Double> ratios = new HashMap<>();
        if (typeFrequencies.isEmpty()) return ratios;
        
        // Calculate total instructions
        int total = typeFrequencies.values().stream().mapToInt(Integer::intValue).sum();
        
        // Calculate ratios
        for (Map.Entry<String, Integer> entry : typeFrequencies.entrySet()) {
            ratios.put(entry.getKey(), entry.getValue() / (double)total);
        }
        
        return ratios;
    }

    // Method to convert ratios to array format for storage
    private float[] convertRatiosToArray(Map<String, Double> ratios) {
        float[] array = new float[InstructionType.values().length];
        for (InstructionType type : InstructionType.values()) {
            array[type.ordinal()] = ratios.getOrDefault(type.name(), 0.0).floatValue();
        }
        return array;
    }
}