Haibao Chen

import inspect
from datetime import datetime, timezone
from typing import Any

from redb.extractors.enum import Tag
from redb.extractors.js_extractor import JSExtractor
from redb.extractors.js_extractors.js_patterns import FEATURE_PATTERNS
from redb.models.dataclasses import JSFeatures


_LONG_STRING_THRESHOLD = 256

# Comments still need a separate scan because the obfuscation metrics consume
# the matched text (to sum its length for comment_ratio), not just its count.
_COMMENT_RE = FEATURE_PATTERNS["comment"]


class JSFeaturesExtractor(JSExtractor):

    def __init__(
        self, filepath, log, exporters=None, index_prefix=None,
        known_benign=False, known_malicious=False, source=None, context=None,
    ):
        super().__init__(
            filepath, log, exporters, index_prefix,
            known_benign, known_malicious, source, context=context,
        )
        self.js_features = None
        self.log.debug(inspect.currentframe().f_code.co_name)

    def tag(self):
        return Tag.JS_FEATURES.value

    @staticmethod
    def _count(scan, name):
        entry = scan.get(name)
        return entry["count"] if entry else 0

    # Score tiers — see docs/js_analysis.md for the rationale behind each
    # threshold. Strong signals are ones that are unambiguous evidence of
    # obfuscation on their own (high encoding density, single-line packers,
    # 1-2 char identifiers). Weak signals are commonly seen in legitimate
    # code (eval, fromCharCode, mid-band entropy) and only count toward the
    # verdict when corroborated.
    _SCORE_STRONG_HEX_DENSITY = 0.05    # >5% of source is \xHH / \uHHHH escapes
    _SCORE_STRONG_MAX_LINE = 5000       # single line ≥5K chars (packer output)
    _SCORE_STRONG_MIN_ID = 2.0          # avg identifier length <2 chars
    _SCORE_STRONG_HIGH_ENTROPY = 5.0    # entropy >5.0 — encoded payload range
    _SCORE_STRONG_NON_ASCII = 0.30      # >30% non-ASCII codepoints in source
    _SCORE_STRONG_UNIQUE_LINES = 0.10   # <10% unique lines (with line_count >100)
    _MIN_LINES_FOR_REPETITION = 100     # below this, repetition isn't meaningful
    # Threshold is paired with the strong-signal gate in the verdict — a high
    # score alone is no longer enough, so the threshold serves as a noise
    # floor, not the false-positive prevention. The gate stops the original
    # "60/100 from weak ticks in clean code" failure mode regardless of where
    # this number sits; 40 keeps the score meaningful without re-banning
    # genuine obfuscation that lacks AST-derived signals (e.g. when
    # pyjsparser isn't installed and avg_identifier_length isn't available).
    _OBFUSCATED_THRESHOLD = 40

    def _score_obfuscation(self, metrics, scan):
        """Return `(score, strong_count, weak_count)` for the obfuscation
        heuristic. The verdict requires `score >= _OBFUSCATED_THRESHOLD` AND
        `strong_count >= 1` (or a js-x-ray hit, handled in the caller); a pile
        of weak signals alone is not enough.
        """
        score = 0
        strong = 0
        weak = 0

        src_len = len(self.js_source) if self.js_source else 1
        line_count = metrics.get('line_count', 1) or 1

        # --- Encoding density (strong / weak split by 1% vs 5%). The old
        # heuristic awarded the same +15 to a sample with 6 hex escapes in
        # 142 KB and to one that was 30% \xHH soup; this fixes that.
        hex_density = (metrics.get('hex_string_count', 0) * 4) / src_len
        unicode_density = (metrics.get('unicode_escape_count', 0) * 6) / src_len
        encoding_density = hex_density + unicode_density
        if encoding_density > self._SCORE_STRONG_HEX_DENSITY:
            score += 20
            strong += 1
        elif encoding_density > 0.01:
            score += 8
            weak += 1

        # --- Identifier length (strong: <2, weak: <3). Obfuscators rename
        # everything to `_0xNNNN` or single chars; legitimate code averages 6+.
        avg_id_len = metrics.get('avg_identifier_length', 10) or 10
        if 0 < avg_id_len < self._SCORE_STRONG_MIN_ID:
            score += 15
            strong += 1
        elif 0 < avg_id_len < 3.0:
            score += 6
            weak += 1

        # --- Single-line packers (strong: >10K, weak: >5K).
        max_line = metrics.get('max_line_length', 0)
        if max_line > 10000:
            score += 15
            strong += 1
        elif max_line > self._SCORE_STRONG_MAX_LINE:
            score += 8
            weak += 1

        # --- Text entropy (strong: >5.0). The old 4.5–5.0 weak band caught
        # jQuery/lodash and is dropped entirely.
        text_entropy = metrics.get('text_entropy', 0)
        if text_entropy > self._SCORE_STRONG_HIGH_ENTROPY:
            score += 15
            strong += 1

        # --- eval (weak; capped at +12). One eval is normal in templating,
        # AngularJS, and polyfills — it can no longer drive 30% of the verdict.
        eval_count = metrics.get('eval_count', 0)
        if eval_count > 0:
            score += min(eval_count * 4, 12)
            weak += 1

        # --- fromCharCode (weak). Common in legacy escapers but worth a tick.
        if metrics.get('fromcharcode_count', 0) > 0:
            score += 6
            weak += 1

        # --- String concatenation density (weak). >20 chains per 100 lines.
        concat_density = self._count(scan, "string_concat") / (line_count / 100)
        if concat_density > 20:
            score += 8
            weak += 1

        # --- Comment-stripped + few-line + large file (weak). Minifier tell.
        comment_ratio = metrics.get('comment_ratio', 0)
        if comment_ratio < 0.01 and line_count < 5 and src_len > 1000:
            score += 5
            weak += 1

        # --- Non-ASCII codepoint density (strong: >30%, weak: >10%). Real-world
        # JS averages <5% non-ASCII (mostly emoji or i18n string literals);
        # ≥30% almost always means a Unicode-codepoint payload (e.g. WSH
        # droppers that build a long string of non-ASCII chars and decode
        # them at runtime). Heavy localization files might cross 30% but
        # typically only score on this signal alone, which can't reach the
        # threshold by itself — the strong-signal gate prevents that
        # false-positive class while still surfacing the case where it
        # corroborates other signals.
        non_ascii_density = metrics.get('non_ascii_density', 0)
        if non_ascii_density > self._SCORE_STRONG_NON_ASCII:
            score += 20
            strong += 1
        elif non_ascii_density > 0.10:
            score += 8
            weak += 1

        # --- Line-uniqueness ratio (strong: <10%, weak: <30%, with
        # line_count >100). Hand-written or even minified code has near-1
        # line uniqueness; <10% means thousands of duplicate lines, which
        # is junk-padding / dead-code-injection used to bloat samples and
        # bury the actual payload. The line-count floor avoids
        # false-positives on tiny files that happen to repeat a few lines.
        unique_line_ratio = metrics.get('unique_line_ratio', 1.0)
        if line_count > self._MIN_LINES_FOR_REPETITION:
            if unique_line_ratio < self._SCORE_STRONG_UNIQUE_LINES:
                score += 15
                strong += 1
            elif unique_line_ratio < 0.30:
                score += 6
                weak += 1

        return min(score, 100), strong, weak

    def _detect_obfuscation_techniques(self, scan, src_len, metrics):
        """Tag the obfuscation techniques present in the source. Densities are
        computed against `src_len` so a handful of escapes in a large file
        does not get the same `hex_encoding` tag as a packed payload.

        Tags mirror the score's signals so the displayed reasoning matches
        the verdict. Three structural tags (`short_identifiers`,
        `packed_single_line`, `high_entropy`) cover the archetypes — minified
        single-line packers, renamed-identifier obfuscators, encoded-payload
        bodies — that the per-API tags below would otherwise miss entirely.
        """
        techniques = []
        if not self.js_source:
            return techniques

        if "eval" in scan:
            techniques.append("eval_usage")
        if "Function constructor" in scan:
            techniques.append("function_constructor")

        # hex_encoding / unicode_encoding by density — match the score's bar
        # so the displayed tags reflect what the score actually credited.
        hex_count = self._count(scan, "hex_escape")
        if hex_count > 5 and (hex_count * 4) / max(src_len, 1) > 0.001:
            techniques.append("hex_encoding")
        unicode_count = self._count(scan, "unicode_escape")
        if unicode_count > 5 and (unicode_count * 6) / max(src_len, 1) > 0.001:
            techniques.append("unicode_encoding")

        # charcode_encoding now requires a real cluster of calls, not one.
        if self._count(scan, "String.fromCharCode") > 3:
            techniques.append("charcode_encoding")
        if self._count(scan, "string_concat") > 10:
            techniques.append("string_concatenation")
        if "atob" in scan:
            techniques.append("base64_decoding")
        if "unescape" in scan:
            techniques.append("unescape_usage")
        if "array_function_call" in scan:
            techniques.append("array_function_calls")

        # Structural tags — surface the score's strong/weak signals so a
        # `is_likely_obfuscated: true` verdict never lands with an empty
        # techniques array (which is what happens on minified packer bodies
        # that don't match any per-API tag above).
        avg_id = metrics.get('avg_identifier_length', 0) or 0
        # Use the weak-tier bar (<3) so both strong (<2) and weak cases
        # surface — `0` means AST was unavailable from both pyjsparser and
        # js-x-ray, so we can't claim anything either way.
        if 0 < avg_id < 3.0:
            techniques.append("short_identifiers")

        # Use the weak-tier bar (>5000) so single-line packers surface even
        # below the strong 10K threshold — both cases credit the score, both
        # deserve a label.
        if metrics.get('max_line_length', 0) > 5000:
            techniques.append("packed_single_line")

        # Mirror the strong-tier entropy bar (>5.0) — the score's only
        # entropy band, since the old 4.5–4.8 weak band was dropped.
        if metrics.get('text_entropy', 0) > 5.0:
            techniques.append("high_entropy")

        # Non-ASCII codepoint payload — Unicode-character buffers that
        # decode at runtime (WSH dropper pattern). Tag at the weak bar
        # (>0.1) so any meaningful presence shows up in the techniques
        # list, even when it's not strong enough on its own.
        if metrics.get('non_ascii_density', 0) > 0.10:
            techniques.append("non_ascii_payload")

        # Repetitive padding — junk-filled bulk that buries the payload
        # under thousands of duplicate lines. Tag at the weak bar (<0.30
        # unique lines) provided the file has enough lines to make the
        # ratio meaningful.
        if (metrics.get('line_count', 0) > self._MIN_LINES_FOR_REPETITION
                and metrics.get('unique_line_ratio', 1.0) < 0.30):
            techniques.append("repetitive_padding")

        return techniques

    def _compute_ast_metrics(self, scan):
        """Compute AST-based metrics: function count, nesting depth, identifiers.

        Falls back to counts from the shared scan dict (function_decl / var_decl
        entries) when pyjsparser is unavailable.
        """
        ast = self._parse_ast()
        if not ast:
            return {
                'total_function_count': self._count(scan, "function_decl"),
                'total_variable_count': self._count(scan, "var_decl"),
                'max_nesting_depth': 0,
                'avg_identifier_length': 0.0,
            }

        counters = {'functions': 0, 'variables': 0}
        identifiers = []
        max_depth = [0]

        def walk(node, depth=0):
            if not isinstance(node, dict):
                return
            node_type = node.get('type', '')
            if node_type in ('FunctionDeclaration', 'FunctionExpression', 'ArrowFunctionExpression'):
                counters['functions'] += 1
            if node_type == 'VariableDeclaration':
                counters['variables'] += len(node.get('declarations', []))
            if node_type == 'Identifier':
                name = node.get('name', '')
                if name:
                    identifiers.append(name)

            # Track nesting depth for blocks/functions
            new_depth = depth
            if node_type in ('BlockStatement', 'FunctionDeclaration', 'FunctionExpression'):
                new_depth = depth + 1
                if new_depth > max_depth[0]:
                    max_depth[0] = new_depth

            for key, value in node.items():
                if key == 'type':
                    continue
                if isinstance(value, dict):
                    walk(value, new_depth)
                elif isinstance(value, list):
                    for item in value:
                        if isinstance(item, dict):
                            walk(item, new_depth)

        try:
            walk(ast)
        except RecursionError:
            self.log.warning(f"AST too deep for {self.hash.sha256}")

        avg_id = 0.0
        if identifiers:
            avg_id = round(sum(len(i) for i in identifiers) / len(identifiers), 2)

        return {
            'total_function_count': counters['functions'],
            'total_variable_count': counters['variables'],
            'max_nesting_depth': max_depth[0],
            'avg_identifier_length': avg_id,
        }

    def extract(self):
        src = self.js_source
        if not src:
            self.log.error(f"Empty JS source for {self.hash.sha256}")
            return None

        lines = self.lines
        line_count = len(lines)
        char_count = len(src)
        # text_entropy is over decoded characters (distinct from BasicProperties.file_entropy
        # over raw bytes); needed for non-ASCII sources where byte entropy is depressed by
        # encoding artefacts (e.g. UTF-16 nulls).
        text_entropy = self._calculate_text_entropy(src)

        line_lengths = [len(l) for l in lines] if lines else [0]
        max_line_length = max(line_lengths)
        avg_line_length = round(sum(line_lengths) / len(line_lengths), 2) if line_lengths else 0.0

        # Minification heuristic: few lines but large file, or very long average lines
        is_minified = (line_count < 5 and char_count > 500) or avg_line_length > 500

        # The shared per-sample scan dict (built once on the JSContext); every
        # count below reads from it, including _detect_obfuscation_techniques()
        # and _score_obfuscation().
        scan = self._context.scan
        # js-x-ray output is also cached on the context — same subprocess runs
        # at most once per sample regardless of how many extractors consult it.
        xray = self._context.xray

        eval_count = self._count(scan, "eval")
        function_constructor_count = self._count(scan, "Function constructor")
        settimeout_setinterval_count = self._count(scan, "settimeout_setinterval")
        document_write_count = self._count(scan, "document.write")
        innerhtml_count = self._count(scan, "innerHTML assignment")
        unescape_count = self._count(scan, "unescape")
        fromcharcode_count = self._count(scan, "String.fromCharCode")
        atob_count = self._count(scan, "atob")
        decodeuri_count = self._count(scan, "decodeURI")

        hex_string_count = self._count(scan, "hex_escape")
        unicode_escape_count = self._count(scan, "unicode_escape")
        long_string_count = self._count(scan, "long_string")
        base64_string_count = self._count(scan, "base64_string")

        # Comment ratio still needs the matched text (to sum its length), so
        # the comment regex is the one pattern we run separately.
        comments = _COMMENT_RE.findall(src)
        comment_chars = sum(len(c) for c in comments)
        comment_ratio = round(comment_chars / char_count, 4) if char_count else 0.0

        # Non-ASCII codepoint density and line-uniqueness ratio — both target
        # patterns the per-API/per-encoding signals miss: Unicode-codepoint
        # payloads (WSH droppers building runtime strings out of >0x7f chars)
        # and junk-padded bulk (thousands of duplicate lines hiding the actual
        # logic). Computed here so they ride alongside the existing metrics.
        non_ascii_count = sum(1 for c in src if ord(c) > 127)
        non_ascii_density = non_ascii_count / char_count if char_count else 0.0
        unique_lines = len({l for l in lines if l.strip()})
        unique_line_ratio = unique_lines / line_count if line_count else 1.0

        ast_metrics = self._compute_ast_metrics(scan)

        # pyjsparser is ES5.1-only — anything with destructuring, classes,
        # optional chaining, etc. fails parse and the AST path returns 0.0
        # for avg_identifier_length, which is exactly the strong signal the
        # heuristic needs to catch obfuscator.io's `_0xNNNN` renaming. js-x-ray
        # parses ES2015+ internally and reports the same statistic, so we use
        # it as the fallback when our own AST is missing.
        avg_id_length = ast_metrics['avg_identifier_length']
        if avg_id_length == 0.0 and xray.avg_identifier_length is not None:
            avg_id_length = xray.avg_identifier_length
            ast_metrics['avg_identifier_length'] = avg_id_length

        metrics = {
            'text_entropy': text_entropy,
            'eval_count': eval_count,
            'hex_string_count': hex_string_count,
            'unicode_escape_count': unicode_escape_count,
            'fromcharcode_count': fromcharcode_count,
            'line_count': line_count,
            'max_line_length': max_line_length,
            'comment_ratio': comment_ratio,
            'avg_identifier_length': avg_id_length,
            'non_ascii_density': non_ascii_density,
            'unique_line_ratio': unique_line_ratio,
        }

        obfuscation_techniques = self._detect_obfuscation_techniques(
            scan, char_count, metrics
        )
        obfuscation_score, strong_signals, _weak = self._score_obfuscation(metrics, scan)

        # Verdict: js-x-ray's recognised obfuscator family is authoritative.
        # Otherwise the heuristic must clear the threshold AND have at least
        # one strong signal — three weak ticks alone are no longer enough.
        obfuscator_name = xray.obfuscator
        is_likely_obfuscated = (
            obfuscator_name is not None
            or (obfuscation_score >= self._OBFUSCATED_THRESHOLD and strong_signals >= 1)
        )

        script_type = self._detect_script_type()
        detected_environment = self._detect_environment()

        self.js_features = JSFeatures(
            line_count=line_count,
            char_count=char_count,
            text_entropy=text_entropy,
            max_line_length=max_line_length,
            avg_line_length=avg_line_length,
            is_minified=is_minified,
            is_likely_obfuscated=is_likely_obfuscated,
            obfuscator_name=obfuscator_name,
            obfuscation_score=obfuscation_score,
            obfuscation_techniques=obfuscation_techniques,
            eval_count=eval_count,
            function_constructor_count=function_constructor_count,
            settimeout_setinterval_count=settimeout_setinterval_count,
            document_write_count=document_write_count,
            innerhtml_count=innerhtml_count,
            unescape_count=unescape_count,
            fromcharcode_count=fromcharcode_count,
            atob_count=atob_count,
            decodeuri_count=decodeuri_count,
            total_function_count=ast_metrics['total_function_count'],
            total_variable_count=ast_metrics['total_variable_count'],
            max_nesting_depth=ast_metrics['max_nesting_depth'],
            avg_identifier_length=ast_metrics['avg_identifier_length'],
            hex_string_count=hex_string_count,
            unicode_escape_count=unicode_escape_count,
            long_string_count=long_string_count,
            base64_string_count=base64_string_count,
            comment_ratio=comment_ratio,
            script_type=script_type,
            detected_environment=detected_environment,
        )
        return self.js_features

    def prepare_export_data(self, exporter_type: str) -> Any:
        if exporter_type == "ClickHouseExporter":
            if not self.js_features:
                return None

            f = self.js_features
            current_time = datetime.now(timezone.utc)
            data = [[
                self.sha256,
                f.line_count,
                f.char_count,
                f.text_entropy,
                f.max_line_length,
                f.avg_line_length,
                int(f.is_minified),
                int(f.is_likely_obfuscated),
                f.obfuscator_name or "",
                f.obfuscation_score,
                f.obfuscation_techniques,
                f.eval_count,
                f.function_constructor_count,
                f.settimeout_setinterval_count,
                f.document_write_count,
                f.innerhtml_count,
                f.unescape_count,
                f.fromcharcode_count,
                f.atob_count,
                f.decodeuri_count,
                f.total_function_count,
                f.total_variable_count,
                f.max_nesting_depth,
                f.avg_identifier_length,
                f.hex_string_count,
                f.unicode_escape_count,
                f.long_string_count,
                f.base64_string_count,
                f.comment_ratio,
                f.script_type,
                f.detected_environment,
                current_time,
            ]]

            column_names = [
                "sha256",
                "line_count", "char_count", "text_entropy",
                "max_line_length", "avg_line_length",
                "is_minified", "is_likely_obfuscated", "obfuscator_name",
                "obfuscation_score", "obfuscation_techniques",
                "eval_count", "function_constructor_count",
                "settimeout_setinterval_count", "document_write_count",
                "innerhtml_count", "unescape_count", "fromcharcode_count",
                "atob_count", "decodeuri_count",
                "total_function_count", "total_variable_count",
                "max_nesting_depth", "avg_identifier_length",
                "hex_string_count", "unicode_escape_count",
                "long_string_count", "base64_string_count",
                "comment_ratio",
                "script_type", "detected_environment",
                "analysis_date",
            ]

            column_type_names = [
                "FixedString(64)",
                "UInt32", "UInt64", "Float64",
                "UInt32", "Float64",
                "UInt8", "UInt8", "LowCardinality(String)",
                "UInt8", "Array(String)",
                "UInt32", "UInt32",
                "UInt32", "UInt32",
                "UInt32", "UInt32", "UInt32",
                "UInt32", "UInt32",
                "UInt32", "UInt32",
                "UInt16", "Float64",
                "UInt32", "UInt32",
                "UInt32", "UInt32",
                "Float64",
                "LowCardinality(String)", "LowCardinality(String)",
                "DateTime64(3, 'UTC')",
            ]

            return (data, column_names, column_type_names)

    def get_clickhouse_table(self) -> str:
        return "redb_js_features"