Haiman Du

import inspect
import pefile
from dotnetfile import DotNetPE

import base64
from hashlib import md5
from pprint import pprint

import magic
import pefile

from redb.extractors.enum import Tag
from redb.extractors.pe_extractor import PEExtractor
from redb.models.dataclasses import PEDotNet
from datetime import datetime, timezone
import json
from typing import Any


class PEDotNetExtractor(PEExtractor):

    def __init__(
        self,
        filepath,
        log,
        exporters=None,
        index_prefix=None,
        elastic_index=None,
        known_benign=False,
        known_malicious=False,
        pe=None,
    ):
        super().__init__(
            filepath,
            log,
            exporters,
            index_prefix,
            elastic_index,
            known_benign,
            known_malicious,
            pe,
        )
        # self.pe_features = None
        self.elastic_index = self.index_prefix + "-pe_dotnet"
        self.dotnet, self.error = self._generate_dotnetfile_object()
        self.log.debug(inspect.currentframe().f_code.co_name)

    def _decode_assembly_flags(self, flag_value):
        self.log.debug(inspect.currentframe().f_code.co_name)
        ASSEMBLY_FLAGS = {
            0x00000001: "COMIMAGE_FLAGS_ILONLY",
            0x00000002: "COMIMAGE_FLAGS_32BITREQUIRED",
            0x00000004: "COMIMAGE_FLAGS_IL_LIBRARY",
            0x00000008: "COMIMAGE_FLAGS_STRONGNAMESIGNED",
            0x00000010: "COMIMAGE_FLAGS_NATIVE_ENTRYPOINT",
            0x00010000: "COMIMAGE_FLAGS_TRACKDEBUGDATA",
            0x00020000: "COMIMAGE_FLAGS_32BITPREFERRED",
        }
        flags = []
        for bit, name in ASSEMBLY_FLAGS.items():
            if flag_value & bit:
                flags.append(name)
        return flags

    def _normalize_data(self, data):
        self.log.debug(inspect.currentframe().f_code.co_name)
        if isinstance(data, str):
            return {"value": data}
        return data

    def _compute_md5(self, data):
        self.log.debug(inspect.currentframe().f_code.co_name)
        if isinstance(data, str):
            # If it's a string, encode it to first
            return md5(data.encode("raw_unicode_escape")).hexdigest()
        elif not isinstance(data, bytes):
            # If it's neither string nor bytes, convert it to string and then to bytes
            return md5(str(data).encode("raw_unicode_escape")).hexdigest()
        else:
            return md5(data).hexdigest()

    def tag(self):
        return Tag.PE_DOTNET.value

    def extract(self):
        self.log.debug(inspect.currentframe().f_code.co_name)
        metadata = {}

        if self.error:
            metadata["Corrupted"] = self.error.args[0]
            dotnet = PEDotNet(metadata)
            self.dotnet = dotnet  # Store for later use in prepare_export_data
            return dotnet  # Return the extracted data instead of exporting directly

        metadata["info"] = {}
        try:
            available_tables = self.dotnet.existent_metadata_tables()
        except Exception as e:
            self.log.error(f"Error getting metadata tables for {self.hash.sha256}: {e}")
            available_tables = []
            metadata["info"]["PotentiallyCorrupted"] = True

        # GUID extraction
        try:
            input_string = self.dotnet.guid_stream_guids[0].string_representation
            
            formatted_uuid = (
                f"{input_string[6:8]}{input_string[4:6]}{input_string[2:4]}{input_string[0:2]}"
                f"-{input_string[10:12]}{input_string[8:10]}"
                f"-{input_string[14:16]}{input_string[12:14]}"
                f"-{input_string[16:20]}"
                f"-{input_string[20:]}"
            )
            metadata["info"]["ModuleVersionID"] = formatted_uuid
        except Exception as e:
            self.log.error(f"Error extracting GUID for {self.hash.sha256}: {e}")

        # CLR Version
        try:
            metadata["info"]["CLRversion"] = self.dotnet.get_runtime_target_version()
        except Exception as e:
            self.log.error(f"Error getting CLR version for {self.hash.sha256}: {e}")

        # Module information
        try:
            if self.dotnet.metadata_table_exists("Module"):
                metadata["info"]["AssemblyModuleName"] = self.dotnet.Module.get_module_name()
        except Exception as e:
            self.log.error(f"Error getting module info for {self.hash.sha256}: {e}")

        # Basic properties
        try:
            metadata["info"].update({
                "IsNativeNgenImage": self.dotnet.is_native_image(),
                "IsMixedAssembly": self.dotnet.is_mixed_assembly(),
                "IsWindowsFormsApp": self.dotnet.is_windows_forms_app(),
                "EntryPointToken": self.dotnet.clr_header.EntryPointToken.value,
                "HasNativeEntryPoint": self.dotnet.has_native_entry_point(),
            })
        except Exception as e:
            self.log.error(f"Error getting basic properties for {self.hash.sha256}: {e}")

        # Assembly flags
        try:
            metadata["info"]["AssemblyFlags"] = self._decode_assembly_flags(
                self.dotnet.clr_header.Flags.value
            )
        except Exception as e:
            self.log.error(f"Error getting assembly flags for {self.hash.sha256}: {e}")

        # More Metadata/Info CLR Header
        try:
            metadata["info"]["NumOfStreams"] = self.dotnet.get_number_of_streams()
            # [TODO] The following two on VT are "RVA entry point"
            #  and "Resources va" respectively. To verify.
            metadata["info"][
                "ResourcesDirectoryAddress"
            ] = self.dotnet.clr_header.ResourcesDirectoryAddress.value
            metadata["info"][
                "ResourcesDirectorySize"
            ] = self.dotnet.clr_header.ResourcesDirectorySize.value
            metadata["info"][
                "StrongNameSignatureAddress"
            ] = self.dotnet.clr_header.StrongNameSignatureAddress.value
            metadata["info"][
                "StrongNameSignatureSize"
            ] = self.dotnet.clr_header.StrongNameSignatureSize.value
            metadata["info"][
                "CodeManagerTableAddress"
            ] = self.dotnet.clr_header.CodeManagerTableAddress.value
            metadata["info"][
                "CodeManagerTableSize"
            ] = self.dotnet.clr_header.CodeManagerTableSize.value
            metadata["info"][
                "VTableFixupsAddress"
            ] = self.dotnet.clr_header.VTableFixupsAddress.value
            metadata["info"][
                "VTableFixupsSize"
            ] = self.dotnet.clr_header.VTableFixupsSize.value
            metadata["info"][
                "ExportAddressTableJumpsAddress"
            ] = self.dotnet.clr_header.ExportAddressTableJumpsAddress.value
            metadata["info"][
                "ExportAddressTableJumpsSize"
            ] = self.dotnet.clr_header.ExportAddressTableJumpsSize.value
            metadata["info"][
                "ManagedNativeHeaderAddress"
            ] = self.dotnet.clr_header.ManagedNativeHeaderAddress.value
            metadata["info"][
                "ManagedNativeHeaderSize"
            ] = self.dotnet.clr_header.ManagedNativeHeaderSize.value
        except Exception as e:
            self.log.error(f"Error getting more metadata/info clr header for {self.hash.sha256}: {e}")

        # Streams section
        try:
            metadata["Streams"] = {}
            stream_names = self.dotnet.get_stream_names()
            for name in stream_names:
                indx = stream_names.index(name)
                stream_header = self.dotnet.dotnet_stream_headers[indx]
                stream_address = self.dotnet.dotnet_streams[indx].address
                metadata["Streams"][name] = {}
                metadata["Streams"][name]["raw_name"] = base64.b64encode(
                    stream_header.Name.value
                ).decode()
                # metadata["Streams"][name]["raw_name"] = name
                metadata["Streams"][name]["size"] = stream_header.Size.value
                # Retrieve the stream data directly using the given address and size
                stream_data = self.pe.get_data(stream_address, stream_header.Size.value)
                metadata["Streams"][name]["entropy"] = (
                    "%.2f" % pefile.SectionStructure.entropy_H(self.pe, stream_data)
                )
                metadata["Streams"][name]["md5"] = md5(stream_data).hexdigest()
        except Exception as e:
            self.log.error(f"Error getting streams for {self.hash.sha256}: {e}")

        # ManifestResource
        try:
            metadata["ManifestResource"] = ()
            if "ManifestResource" in available_tables:
                metadata["info"]["HasManifestResource"] = True
                metadata["ManifestResource"] = (
                    self.dotnet.ManifestResource.get_resource_names()
                )
        except Exception as e:
            self.log.error(f"Error getting manifest resource for {self.hash.sha256}: {e}")

        # ExternalAssemblyRef
        try:
            metadata["ExternalAssemblyRef"] = {}
            if "AssemblyRef" in available_tables:
                metadata["ExternalAssemblyRef"][
                    "names"
                ] = self.dotnet.AssemblyRef.get_assemblyref_names()
                metadata["ExternalAssemblyRef"][
                    "cultures"
                ] = self.dotnet.AssemblyRef.get_assemblyref_cultures()
        except Exception as e:
            self.log.error(f"Error getting external assembly ref for {self.hash.sha256}: {e}")

        # AssemblyData
        try:
            metadata["AssemblyData"] = {}
            if "Assembly" in available_tables:
                metadata["AssemblyData"][
                    "name"
                ] = self.dotnet.Assembly.get_assembly_name()
                # It's officially a one-row table, but there are ITW files with more than one row.
                # It stores information about the current assembly. It is documented in ECMA-335.
                assembly_table = self.dotnet.metadata_tables_lookup[
                    "Assembly"
                ].table_rows[0]
                metadata["AssemblyData"][
                    "MajorVersion"
                ] = assembly_table.MajorVersion.value
                metadata["AssemblyData"][
                    "MinorVersion"
                ] = assembly_table.MinorVersion.value
                metadata["AssemblyData"]["hashalgid"] = assembly_table.HashAlgId.value
                metadata["AssemblyData"][
                    "BuildNumber"
                ] = assembly_table.BuildNumber.value
                metadata["AssemblyData"][
                    "RevisionNumber"
                ] = assembly_table.RevisionNumber.value
                metadata["AssemblyData"][
                    "culture"
                ] = self.dotnet.Assembly.get_assembly_culture()
        except Exception as e:
            self.log.error(f"Error getting assembly data for {self.hash.sha256}: {e}")

        # TypeDef handling
        try:
            metadata["TypeDef"] = {}
            if "TypeDef" in available_tables:
                metadata["info"]["HasTypeDef"] = True
                # Get basic type definitions
                try:
                    # Get all types (ANY visibility)
                    type_defs = self.dotnet.TypeDef.get_type_names()
                    metadata["TypeDef"]["names"] = type_defs if type_defs else []
                    
                    # Get detailed type information with methods
                    types_with_methods = self.dotnet.TypeDef.get_type_names_with_methods()
                    if types_with_methods:
                        metadata["TypeDef"]["detailed"] = []
                        for type_info in types_with_methods:
                            type_data = {
                                "type": type_info.Type,
                                "namespace": type_info.Namespace,
                                "flags": type_info.Flags,
                                "methods": []
                            }
                            for method in type_info.Methods:
                                method_data = {
                                    "name": method.Name,
                                    "flags": method.Flags
                                }
                                if method.Signature:
                                    method_data["signature"] = {
                                        "parameters": method.Signature.get("parameter", []),
                                        "return_type": method.Signature.get("return", ""),
                                        "has_this": method.Signature.get("hasthis", False)
                                    }
                                type_data["methods"].append(method_data)
                            metadata["TypeDef"]["detailed"].append(type_data)
                except AttributeError as e:
                    self.log.warning(f"TypeDef table exists but attributes missing for {self.hash.sha256}: {e}")
                    # metadata["info"]["TypeDefParsingError"] = f"Missing attributes: {str(e)}"
                except Exception as e:
                    self.log.error(f"Error parsing TypeDef table for {self.hash.sha256}: {e}")
                    # metadata["info"]["TypeDefParsingError"] = str(e)
        except Exception as e:
            self.log.error(f"Error handling TypeDef section for {self.hash.sha256}: {e}")
            metadata["TypeDef"] = {"names": [], "detailed": []}
            # metadata["info"]["TypeDefParsingError"] = str(e)

        # ExternalUnmanagedModules
        try:
            metadata["ExternalUnmanagedModules"] = ()
            if "ModuleRef" in available_tables:
                metadata["info"]["HasModuleRef"] = True
                metadata["ExternalUnmanagedModules"] = (
                    self.dotnet.ModuleRef.get_unmanaged_module_names(
                        self.dotnet.Type.UnmanagedModules.RAW
                    )
                )
        except Exception as e:
            self.log.error(f"Error getting external unmanaged modules for {self.hash.sha256}: {e}")

        # ExternalUnmanagedMethods
        try:
            metadata["ExternalUnmanagedMethods"] = ()
            if "ImplMap" in available_tables:
                metadata["info"]["HasImplMap"] = True
                metadata["ExternalUnmanagedMethods"] = (
                    self.dotnet.ImplMap.get_unmanaged_functions()
                )
        except Exception as e:
            self.log.error(f"Error getting external unmanaged methods for {self.hash.sha256}: {e}")

        # Event
        try:
            metadata["Event"] = ()
            if "Event" in available_tables:
                metadata["info"]["HasEvent"] = True
                metadata["Event"] = self.dotnet.Event.get_event_names()
        except Exception as e:
            self.log.error(f"Error getting event for {self.hash.sha256}: {e}")

        # Resources
        try:
            metadata["Resources"] = []
            if self.dotnet.has_resources():
                tmp_resources_list = []
                resource_data = self.dotnet.get_resources()
                for data in resource_data:
                    tmp_dict = {}
                    for resource_item in data.items():
                        if resource_item[0] == "SubResources":
                            if resource_item[1]:
                                tmp_dict["SubResources"] = {}
                                for sub_resource in resource_item[1]:
                                    for sub_resource_item in sub_resource.items():
                                        if sub_resource_item[0] == "Data":
                                            # Check if data is a sequence type that supports len()
                                            if hasattr(sub_resource_item[1], '__len__') and len(sub_resource_item[1]) > 100:
                                                tmp_dict["SubResources"]["Data"] = {}
                                                try:
                                                    # tmp_dict["SubResources"]["Data"][
                                                    #     "md5"
                                                    # ] = md5(
                                                    #     sub_resource_item[1].encode()
                                                    # ).hexdigest()
                                                    tmp_dict["SubResources"]["Data"][
                                                        "md5"
                                                    ] = self._compute_md5(
                                                        sub_resource_item[1]
                                                    )
                                                except Exception as e:
                                                    self.log.error(
                                                        f"Error in dotnet resources"
                                                        f" extraction {self.hash.sha256}: {e}"
                                                    )
                                                guess = magic.from_buffer(
                                                    sub_resource_item[1], mime=True
                                                )
                                                tmp_dict["SubResources"]["Data"][
                                                    "MimeType"
                                                ] = (guess if guess else None)
                                            else:
                                                normalized_data = self._normalize_data(
                                                    sub_resource_item[1].decode()
                                                    if isinstance(
                                                        sub_resource_item[1], bytes
                                                    )
                                                    else sub_resource_item[1]
                                                )
                                                tmp_dict["SubResources"][
                                                    sub_resource_item[0]
                                                ] = normalized_data
                                        else:
                                            tmp_dict["SubResources"][
                                                sub_resource_item[0]
                                            ] = (
                                                sub_resource_item[1].decode()
                                                if isinstance(
                                                    sub_resource_item[1], bytes
                                                )
                                                else sub_resource_item[1]
                                            )
                        elif resource_item[0] == "Data":
                            # Check if data is a sequence type that supports len()
                            if hasattr(resource_item[1], '__len__') and len(resource_item[1]) > 100:
                                tmp_dict["Data"] = {}
                                tmp_dict["Data"]["md5"] = self._compute_md5(
                                    resource_item[1]
                                )
                                guess = magic.from_buffer(resource_item[1], mime=True)
                                tmp_dict["Data"]["MimeType"] = guess if guess else None
                            else:
                                normalized_data = self._normalize_data(
                                    resource_item[1].decode()
                                    if isinstance(resource_item[1], bytes)
                                    else resource_item[1]
                                )
                                tmp_dict["Data"] = normalized_data
                        else:
                            tmp_dict[resource_item[0]] = (
                                resource_item[1].decode()
                                if isinstance(resource_item[1], bytes)
                                else resource_item[1]
                            )
                    tmp_resources_list.append(tmp_dict)
                metadata["Resources"] = tmp_resources_list
        except Exception as e:
            self.log.error(f"Error getting resources for {self.hash.sha256}: {e}")
            metadata["Resources"] = []  # Ensure we always have a valid list even on error

        dotnet = PEDotNet(metadata)
        self.dotnet = dotnet  # Store for later use in prepare_export_data
        return dotnet  # Return the extracted data instead of exporting directly


    def prepare_export_data(self, exporter_type: str) -> Any:
        if exporter_type == "ElasticsearchExporter":
            return self.dotnet
        elif exporter_type == "ClickHouseExporter":
            # Convert metadata to JSON string
            metadata_json = json.dumps(self.dotnet.dotnet)
            
            data = [[
                self.sha256,
                self.md5,
                self.sha1,
                metadata_json,
                datetime.now(timezone.utc)
            ]]
            
            column_names = [
                'sha256', 'md5', 'sha1', 'dotnet', 'analysis_date'
            ]
            
            if not data:
                return None

            column_type_names = [
                'String', 'String', 'String', 'JSON', 'DateTime64(3, \'UTC\')'
            ]

            return (data, column_names, column_type_names)

    def get_clickhouse_table(self) -> str:
        return "redb_pe_dotnet"