Hanako Yoshida

23 papers B 11Journal 2Unranked 10
YearRankTypeTitle / Venue / Authors
2025 B conf
CogSci
Elizsbeth Perkovich, Hanako Yoshida
2025 B conf
CogSci
Elizsbeth Perkovich, Hanako Yoshida
2024 conf
ICDL
Lichao Sun, Hanako Yoshida
2024 conf
ICDL
Elizsbeth Perkovich, Hanako Yoshida
2024 conf
ICDL
Elizsbeth Perkovich, Hanako Yoshida
2023 conf
ICDL
Lichao Sun, Hanako Yoshida
2023 conf
ICDL
Lichao Sun, Nikita Gidh, Hanako Yoshida
2016 J jnl
CoRR
Qazaleh Mirsharif, Sidharth Sadani, Shishir K. Shah, Hanako Yoshida, Joseph M. Burling
2016 conf
CVIP (2)
Qazaleh Mirsharif, Sidharth Sadani, Shishir K. Shah, Hanako Yoshida, Joseph M. Burling
2015 B conf
CogSci
Lars Schillingmann, Joseph M. Burling, Hanako Yoshida, Yukie Nagai
2014 B conf
CogSci
Samuel Bilson, Hanako Yoshida, Thomas T. Hills
2014 conf
ICDL-EPIROB
Beata J. Grzyb, Allegra Cattani, Angelo Cangelosi, Caroline Floccia, Hanako Yoshida, Joseph M. Burling, Anna M. Borghi, Swapnaa Jayaraman, Linda B. Smith, Alfredo F. Pereira, Isabel C. Lisboa, Emanuel Sousa, Jorge A. Santos, Wolfram Erlhagen, Estela Bicho
2013 B conf
CogSci
Hanako Yoshida, Joseph M. Burling
2013 conf
ICDL-EPIROB
Hanako Yoshida, Joseph M. Burling
2013 conf
ICDL-EPIROB
Joseph M. Burling, Hanako Yoshida, Yukie Nagai
2012 B conf
CogSci
Elizabeth Woods, Hanako Yoshida
2012 B conf
CogSci
Duc Tran, Hanako Yoshida
2012 B conf
CogSci
Joseph M. Burling, Hanako Yoshida
2012 conf
IEEE CLOUD
Jong-Hyuk Lee, Tao Feng, Weidong Shi, Apurva Bedagkar-Gala, Shishir K. Shah, Hanako Yoshida
2011 B conf
CogSci
Joseph M. Burling, Hanako Yoshida
2011 B conf
CogSci
Hanako Yoshida, Kevin Darby, Joseph M. Burling
2011 B conf
CogSci
Duc Tran, Hanako Yoshida
2010 J jnl
Cogn. Sci.
Linda B. Smith, Eliana Colunga, Hanako Yoshida
notebooks/redb-clickhouse-TEST.ipynb
← Index notebooks/redb-clickhouse-TEST.ipynb plaintext
Truncated preview: showing the first 16384 bytes out of 5988848 total.
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import clickhouse_connect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "ch_TEST_client = clickhouse_connect.get_client(host='e.rationaledge.io', port=443, username='default', password='werioubTGwvf0-dsbv_', database='redb', verify=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ch_TEST_BETA_client = clickhouse_connect.get_client(host='k.rationaledge.io', port=443, username='default', password='werioubTGwvf0-dsbv_', database='default', verify=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method HttpClient.close_connections of <clickhouse_connect.driver.httpclient.HttpClient object at 0x105fd64d0>>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ch_TEST_BETA_client.close_connections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<clickhouse_connect.driver.summary.QuerySummary at 0x11f900810>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ch_TEST_BETA_client.command(\"\"\"\n",
    "RENAME DATABASE redb TO orig_redb;\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DB Creation (schema 2025-08-02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ch_TEST_client.command(\"\"\"\n",
    "CREATE DATABASE IF NOT EXISTS redb;\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define General Tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### CREATE FULL REDB DATABASE\n",
    "def create_redb_with_all_tables(client):\n",
    "    client.command(\"\"\"\n",
    "        CREATE DATABASE IF NOT EXISTS redb\n",
    "    \"\"\")\n",
    "    create_all_general_tables(client)\n",
    "    create_all_pe_tables(client)\n",
    "\n",
    "#### CREATE FULL TABLES\n",
    "def create_all_general_tables(client):\n",
    "    create_basic_properties_dbtable(client)\n",
    "    create_hashes_dbtable(client)\n",
    "    create_capa_dbtable(client)\n",
    "    create_die_dbtable(client)\n",
    "    create_extra_findings_dbtable(client)\n",
    "\n",
    "\n",
    "##################################\n",
    "#### CREATE INDIVIDUAL TABLES ####\n",
    "##################################\n",
    "\n",
    "#### GENERAL TABLES ####\n",
    "# All tables have SETTINGS index_granularity = 8192\n",
    "def create_basic_properties_dbtable(client):\n",
    "    # Create samples_metadata table\n",
    "    client.command(\"\"\"\n",
    "        CREATE TABLE IF NOT EXISTS redb_basic_properties (\n",
    "            sha256 FixedString(64), -- primary key\n",
    "            md5 FixedString(32),\n",
    "            sha1 FixedString(40),\n",
    "            filename Nullable(String),\n",
    "            sample_name String,\n",
    "            filesize UInt64,\n",
    "            file_entropy Float64,\n",
    "            filetype LowCardinality(String),\n",
    "            filetype_mime LowCardinality(String),\n",
    "            filetype_magika LowCardinality(String),\n",
    "            analysis_date DateTime64(3, 'UTC'),\n",
    "        ) ENGINE = ReplacingMergeTree(analysis_date)\n",
    "        PRIMARY KEY (sha256)\n",
    "        ORDER BY (sha256)\n",
    "    \"\"\")\n",
    "\n",
    "def create_hashes_dbtable(client):\n",
    "    # Create hashes table\n",
    "    client.command(\"\"\"\n",
    "        CREATE TABLE IF NOT EXISTS redb_hashes (\n",
    "            sha256 FixedString(64), -- primary key\n",
    "            md5 FixedString(32),\n",
    "            sha1 FixedString(40),\n",
    "            ssdeep_hash Nullable(String),\n",
    "            tlsh_hash Nullable(String),\n",
    "            authentihash Nullable(String),\n",
    "            imphash Nullable(String),\n",
    "            impfuzzy Nullable(String),\n",
    "            typerefhash Nullable(String),\n",
    "            richhash Nullable(String),\n",
    "            richpe_hash Nullable(String),\n",
    "            richpv_hash Nullable(String),\n",
    "            richpv_hash_sorted Nullable(String),\n",
    "            dhash Nullable(String),\n",
    "            analysis_date DateTime64(3, 'UTC')\n",
    "        ) ENGINE = ReplacingMergeTree(analysis_date)\n",
    "        PRIMARY KEY (sha256)\n",
    "        ORDER BY (sha256)\n",
    "    \"\"\")\n",
    "\n",
    "def create_capa_dbtable(client):\n",
    "    # Create sample_capabilities table\n",
    "    client.command(\"\"\"\n",
    "        SET enable_json_type = 1;\n",
    "    \"\"\")\n",
    "    client.command(\"\"\"\n",
    "        -- Base table for CAPA analysis results\n",
    "        CREATE TABLE IF NOT EXISTS redb_capa (\n",
    "            sha256 FixedString(64),\n",
    "            md5 FixedString(32),\n",
    "            sha1 FixedString(40),\n",
    "            analysis_date DateTime64(3, 'UTC'),\n",
    "            -- Store full JSON dumps as strings for flexibility\n",
    "            capa_dump JSON CODEC(ZSTD(3)),\n",
    "            attack_dump JSON CODEC(ZSTD(3)),\n",
    "            mbc_dump JSON CODEC(ZSTD(3))\n",
    "        ) ENGINE = ReplacingMergeTree(analysis_date)\n",
    "        ORDER BY sha256;\n",
    "    \"\"\")\n",
    "\n",
    "    client.command(\"\"\"\n",
    "        -- Normalized table for capabilities and namespaces\n",
    "        CREATE TABLE IF NOT EXISTS redb_capa_capabilities (\n",
    "            sha256 FixedString(64),\n",
    "            md5 FixedString(32),\n",
    "            sha1 FixedString(40),\n",
    "            analysis_date DateTime64(3, 'UTC'),\n",
    "            namespace LowCardinality(String),\n",
    "            capability LowCardinality(String)\n",
    "        ) ENGINE = ReplacingMergeTree(analysis_date)\n",
    "        ORDER BY (sha256, namespace, capability);\n",
    "    \"\"\")\n",
    "\n",
    "    client.command(\"\"\"\n",
    "        -- Materialized view for capability statistics\n",
    "        CREATE MATERIALIZED VIEW IF NOT EXISTS redb_capa_capabilities_stats (\n",
    "            namespace LowCardinality(String),\n",
    "            capability LowCardinality(String),\n",
    "            samples_bitmap AggregateFunction(groupBitmap, UInt64),\n",
    "            occurrence_count UInt64\n",
    "        ) ENGINE = AggregatingMergeTree()\n",
    "        ORDER BY (namespace, capability)\n",
    "        AS SELECT\n",
    "            namespace,\n",
    "            capability,\n",
    "            groupBitmapState(cityHash64(sha256)) as samples_bitmap,\n",
    "            count() as occurrence_count\n",
    "        FROM redb_capa_capabilities\n",
    "        GROUP BY namespace, capability;\n",
    "    \"\"\")\n",
    "\n",
    "    client.command(\"\"\"\n",
    "        -- Normalized table for MITRE ATT&CK mappings\n",
    "        CREATE TABLE IF NOT EXISTS redb_capa_attack (\n",
    "            sha256 FixedString(64),\n",
    "            md5 FixedString(32),\n",
    "            sha1 FixedString(40),\n",
    "            analysis_date DateTime64(3, 'UTC'),\n",
    "            tactic LowCardinality(String),\n",
    "            technique LowCardinality(String),\n",
    "            technique_id LowCardinality(String)\n",
    "        ) ENGINE = ReplacingMergeTree(analysis_date)\n",
    "        ORDER BY (sha256, tactic, technique_id);\n",
    "    \"\"\")\n",
    "\n",
    "    client.command(\"\"\"\n",
    "        -- Materialized view for ATT&CK statistics\n",
    "        CREATE MATERIALIZED VIEW IF NOT EXISTS redb_capa_attack_stats (\n",
    "            tactic LowCardinality(String),\n",
    "            technique LowCardinality(String),\n",
    "            technique_id LowCardinality(String),\n",
    "            samples_bitmap AggregateFunction(groupBitmap, UInt64),\n",
    "            occurrence_count UInt64\n",
    "        ) ENGINE = AggregatingMergeTree()\n",
    "        ORDER BY (tactic, technique_id)\n",
    "        AS SELECT\n",
    "            tactic,\n",
    "            technique,\n",
    "            technique_id,\n",
    "            groupBitmapState(cityHash64(sha256)) as samples_bitmap,\n",
    "            count() as occurrence_count\n",
    "        FROM redb_capa_attack\n",
    "        GROUP BY tactic, technique, technique_id;\n",
    "    \"\"\")\n",
    "\n",
    "    client.command(\"\"\"\n",
    "        -- Normalized table for MBC mappings\n",
    "        CREATE TABLE IF NOT EXISTS redb_capa_mbc (\n",
    "            sha256 FixedString(64),\n",
    "            md5 FixedString(32),\n",
    "            sha1 FixedString(40),\n",
    "            analysis_date DateTime64(3, 'UTC'),\n",
    "            objective LowCardinality(String),\n",
    "            behavior LowCardinality(String),\n",
    "            behavior_id LowCardinality(String)\n",
    "        ) ENGINE = ReplacingMergeTree(analysis_date)\n",
    "        ORDER BY (sha256, objective, behavior_id);\n",
    "    \"\"\")\n",
    "\n",
    "    client.command(\"\"\"\n",
    "        -- Materialized view for MBC statistics\n",
    "        CREATE MATERIALIZED VIEW IF NOT EXISTS redb_capa_mbc_stats (\n",
    "            objective LowCardinality(String),\n",
    "            behavior LowCardinality(String),\n",
    "            behavior_id LowCardinality(String),\n",
    "            samples_bitmap AggregateFunction(groupBitmap, UInt64),\n",
    "            occurrence_count UInt64\n",
    "        ) ENGINE = AggregatingMergeTree()\n",
    "        ORDER BY (objective, behavior_id)\n",
    "        AS SELECT\n",
    "            objective,\n",
    "            behavior,\n",
    "            behavior_id,\n",
    "            groupBitmapState(cityHash64(sha256)) as samples_bitmap,\n",
    "            count() as occurrence_count\n",
    "        FROM redb_capa_mbc\n",
    "        GROUP BY objective, behavior, behavior_id;\n",
    "    \"\"\")\n",
    "\n",
    "def create_die_dbtable(client):\n",
    "    # Create sample_die table\n",
    "    client.command(\"\"\"\n",
    "        SET enable_json_type = 1;\n",
    "    \"\"\")\n",
    "    client.command(\"\"\"\n",
    "        CREATE TABLE IF NOT EXISTS redb_die (\n",
    "            sha256 FixedString(64),\n",
    "            md5 FixedString(32),\n",
    "            sha1 FixedString(40),\n",
    "            die_info JSON CODEC(ZSTD(3)),\n",
    "            analysis_date DateTime64(3, 'UTC')\n",
    "        ) ENGINE = ReplacingMergeTree(analysis_date)\n",
    "        PRIMARY KEY (sha256)\n",
    "        ORDER BY (sha256)\n",
    "    \"\"\")\n",
    "\n",
    "    client.command(\"\"\"\n",
    "        CREATE MATERIALIZED VIEW redb_die_mv\n",
    "        (\n",
    "            sha256 FixedString(64),\n",
    "            md5 FixedString(32),\n",
    "            sha1 FixedString(40),\n",
    "            analysis_date DateTime64(3, 'UTC'),\n",
    "            compiler LowCardinality(Nullable(String)),\n",
    "            compiler_full Nullable(String),\n",
    "            format LowCardinality(Nullable(String)),\n",
    "            format_full Nullable(String),\n",
    "            installer LowCardinality(Nullable(String)),\n",
    "            installer_full Nullable(String),\n",
    "            joiner LowCardinality(Nullable(String)),\n",
    "            joiner_full Nullable(String),\n",
    "            language LowCardinality(Nullable(String)),\n",
    "            language_full Nullable(String),\n",
    "            library LowCardinality(Nullable(String)),\n",
    "            library_full Nullable(String),\n",
    "            linker LowCardinality(Nullable(String)),\n",
    "            linker_full Nullable(String),\n",
    "            operation_system LowCardinality(Nullable(String)),\n",
    "            operation_system_full Nullable(String),\n",
    "            packer LowCardinality(Nullable(String)),\n",
    "            packer_full Nullable(String),\n",
    "            protector LowCardinality(Nullable(String)),\n",
    "            protector_full Nullable(String),\n",
    "            sfx LowCardinality(Nullable(String)),\n",
    "            sfx_full Nullable(String),\n",
    "            sign_tool LowCardinality(Nullable(String)),\n",
    "            sign_tool_full Nullable(String),\n",
    "            stub LowCardinality(Nullable(String)),\n",
    "            stub_full Nullable(String),\n",
    "            tool LowCardinality(Nullable(String)),\n",
    "            tool_full Nullable(String),   \n",
    "            -- overlay JSON CODEC(ZSTD(3)),\n",
    "            -- resource JSON CODEC(ZSTD(3)),\n",
    "            has_overlay UInt8,\n",
    "            has_resources UInt8,\n",
    "            is_packed UInt8\n",
    "        )\n",
    "        ENGINE = ReplacingMergeTree(analysis_date)\n",
    "        ORDER BY (sha256)\n",
    "        POPULATE\n",
    "        AS SELECT\n",
    "            sha256,\n",
    "            md5,\n",
    "            sha1,\n",
    "            analysis_date,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'compiler'), '') AS compiler,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'compiler(full)'), '') AS compiler_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'format'), '') AS format,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'format(full)'), '') AS format_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'installer'), '') AS installer,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'installer(full)'), '') AS installer_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'joiner'), '') AS joiner,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'joiner(full)'), '') AS joiner_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'language'), '') AS language,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'language(full)'), '') AS language_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'library'), '') AS library,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'library(full)'), '') AS library_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'linker'), '') AS linker,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'linker(full)'), '') AS linker_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'operation_system'), '') AS operation_system,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'operation_system(full)'), '') AS operation_system_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'packer'), '') AS packer,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'packer(full)'), '') AS packer_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'protector'), '') AS protector,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'protector(full)'), '') AS protector_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'sfx'), '') AS sfx,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'sfx(full)'), '') AS sfx_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'sign_tool'), '') AS sign_tool,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'sign_tool(full)'), '') AS sign_tool_full,\n",
    "            nullIf(JSONExtractString(toString(die_info), 'stub'), '') AS stub,\n",
    "            nullIf(JSONExtractString(toString(die_info),