Wei Di

25 papers A* 4A 3B 2C 4Journal 10Unranked 2
YearRankTypeTitle / Venue / Authors
2018 C conf
IGARSS
Wei Di, Changlin Chen, Yongxiang Liu
2015 C conf
ICVS
Kevin Shih, Wei Di, Vignesh Jagadeesh, Robinson Piramuthu
2015 A* conf
ICCV
Zhicheng Yan, Hao Zhang, Robinson Piramuthu, Vignesh Jagadeesh, Dennis DeCoste, Wei Di, Yizhou Yu
2015 B conf
ICIP
M. Hadi Kiapour, Wei Di, Vignesh Jagadeesh, Robinson Piramuthu
2014 B conf
ICIP
Rohit Pandey, Wei Di, Vignesh Jagadeesh, Robinson Piramuthu, Anurag Bhardwaj
2014 J jnl
CoRR
Kevin Shih, Wei Di, Vignesh Jagadeesh, Robinson Piramuthu
2014 J jnl
CoRR
Anurag Bhardwaj, Vignesh Jagadeesh, Wei Di, Robinson Piramuthu, Elizabeth F. Churchill
2014 A conf
WACV
Vicente Ordonez, Vignesh Jagadeesh, Wei Di, Anurag Bhardwaj, Robinson Piramuthu
2014 J jnl
CoRR
Zixuan Wang, Wei Di, Anurag Bhardwaj, Vignesh Jagadeesh, Robinson Piramuthu
2014 J jnl
CoRR
Zhicheng Yan, Vignesh Jagadeesh, Dennis DeCoste, Wei Di, Robinson Piramuthu
2014 A conf
WACV
Mohammad Haris Baig, Vignesh Jagadeesh, Robinson Piramuthu, Anurag Bhardwaj, Wei Di, Neel Sundaresan
2014 A conf
WSDM
Wei Di, Neel Sundaresan, Robinson Piramuthu, Anurag Bhardwaj
2014 J jnl
CoRR
Vignesh Jagadeesh, Robinson Piramuthu, Anurag Bhardwaj, Wei Di, Neel Sundaresan
2014 A* conf
KDD
Vignesh Jagadeesh, Robinson Piramuthu, Anurag Bhardwaj, Wei Di, Neel Sundaresan
2014 A* conf
CVPR
Chen-Yu Lee, Anurag Bhardwaj, Wei Di, Vignesh Jagadeesh, Robinson Piramuthu
2014 J jnl
CoRR
Wei Di, Anurag Bhardwaj, Vignesh Jagadeesh, Robinson Piramuthu, Elizabeth F. Churchill
2013 A* conf
KDD
Anurag Bhardwaj, Atish Das Sarma, Wei Di, Raffay Hamid, Robinson Piramuthu, Neel Sundaresan
2013 conf
CVPR Workshops
Wei Di, Catherine Wah, Anurag Bhardwaj, Robinson Piramuthu, Neel Sundaresan
2012 J jnl
IEEE Trans. Geosci. Remote. Sens.
Wei Di, Melba M. Crawford
2011 J jnl
IEEE J. Sel. Top. Signal Process.
Wei Di, Melba M. Crawford
2011 C conf
IGARSS
Wei Di, Melba M. Crawford
2010 conf
WHISPERS
Wei Di, Melba M. Crawford
2010 C conf
IGARSS
Wei Di, Melba M. Crawford
2010 J jnl
IEEE Trans. Syst. Man Cybern. Part A
Wei Di, Lei Zhang, David Zhang, Quan Pan
2008 J jnl
Pattern Recognit. Lett.
Lin He, Quan Pan, Wei Di, Yuanqing Li
redb/s3_utils.py
← Index redb/s3_utils.py python
"""
S3 and file utility functions for REDB.

Contains MinIO/S3 client creation, S3 key generation,
file download, and archive extraction helpers.
"""
import os
import tempfile
from typing import Optional, List

from minio import Minio
from minio.error import S3Error


def get_minio_client():
    """Create and return a Minio client using credentials from .env file"""
    return Minio(
        endpoint=os.getenv("S3_ENDPOINT"),
        access_key=os.getenv("S3_ACCESS_KEY"),
        secret_key=os.getenv("S3_SECRET_KEY"),
        secure=os.getenv("S3_SECURE", "false").lower() == "true"
    )


def generate_s3_key_from_hash(sample_hash: str) -> str:
    """
    Generate S3 key from hash using sharding structure.
    Example: 09f7d02a3c2382199458c98a62b045145ee54ab6aba86166aecf3d10c3c1444c
    becomes: 09/f7/09f7d02a3c2382199458c98a62b045145ee54ab6aba86166aecf3d10c3c1444c.zip
    """
    if len(sample_hash) < 4:
        raise ValueError(f"Hash too short for sharding: {sample_hash}")

    # Create sharded path: first 2 chars / next 2 chars / full_hash.zip
    return f"{sample_hash[:2]}/{sample_hash[2:4]}/{sample_hash}.zip"


def download_s3_object(s3_bucket: str, s3_key: str, temp_dir: str, logger=None) -> Optional[str]:
    """
    Download an object from S3 to a local temporary directory and return the local path.
    If the .zip file is not found, automatically tries .7z as a fallback for legacy uploads.
    """
    try:
        minio_client = get_minio_client()
        local_path = os.path.join(temp_dir, os.path.basename(s3_key))

        # Download the file
        minio_client.fget_object(s3_bucket, s3_key, local_path)

        return local_path
    except S3Error as e:
        # If .zip not found and error is NoSuchKey, try .7z fallback
        if e.code == 'NoSuchKey' and s3_key.endswith('.zip'):
            s3_key_7z = s3_key[:-4] + '.7z'  # Replace .zip with .7z
            if logger:
                logger.info(f".zip not found, trying .7z fallback: {s3_key_7z}")
            try:
                local_path_7z = os.path.join(temp_dir, os.path.basename(s3_key_7z))
                minio_client.fget_object(s3_bucket, s3_key_7z, local_path_7z)
                if logger:
                    logger.info(f"Successfully downloaded .7z fallback: {s3_key_7z}")
                return local_path_7z
            except S3Error as e2:
                if logger:
                    logger.error(f"Failed to download S3 object {s3_key} (tried both .zip and .7z): {e2}")
                else:
                    print(f"[ERROR] Failed to download S3 object {s3_key} (tried both .zip and .7z): {e2}")
                return None
        else:
            if logger:
                logger.error(f"Failed to download S3 object {s3_key}: {e}")
            else:
                print(f"[ERROR] Failed to download S3 object {s3_key}: {e}")
            return None


def extract_fat_slices(macho, output_dir: str, logger) -> List[tuple]:
    """
    Extract slices from FAT Mach-O binary using machofile's dump_slices() API.

    Args:
        macho: Parsed machofile.UniversalMachO object
        output_dir: Directory to write slice files
        logger: Logger instance

    Returns:
        List of (arch_name, slice_path, slice_sha256, slice_md5, slice_sha1) tuples
    """
    result = []
    try:
        # Use machofile's dump_slices API (v2026.2.5+)
        slices = macho.dump_slices(output_dir=output_dir)
        logger.debug(f"Extracted {len(slices)} slices from FAT binary")

        for arch_name, slice_path in slices:
            # Get hash info for this slice from machofile
            general_info = macho.get_general_info(arch=arch_name)
            slice_sha256 = general_info.get('SHA256')
            slice_md5 = general_info.get('MD5')
            slice_sha1 = general_info.get('SHA1')
            result.append((arch_name, slice_path, slice_sha256, slice_md5, slice_sha1))
            logger.debug(f"Slice {arch_name}: {slice_sha256}")

    except Exception as e:
        logger.error(f"Error extracting FAT slices: {e}")

    return result