3.2 图像特征提取与检索（续）

3.2.3 图像检索技术

基于特征的图像检索

import faiss
import numpy as np
from typing import List, Tuple

class FeatureBasedImageRetrieval:
    def __init__(self, feature_dim: int = 2048):
        self.feature_dim = feature_dim
        self.index = None
        self.image_paths = []
    
    def build_index(self, features: np.ndarray, image_paths: List[str]):
        """构建图像检索索引"""
        self.image_paths = image_paths
        
        # 创建FAISS索引
        self.index = faiss.IndexFlatL2(self.feature_dim)
        
        # 添加特征到索引
        features = features.astype(np.float32)
        self.index.add(features)
    
    def search(self, query_features: np.ndarray, top_k: int = 10) -> List[Tuple[str, float]]:
        """搜索相似图像"""
        if self.index is None:
            raise ValueError("索引未构建")
        
        # 归一化查询特征
        query_features = query_features.astype(np.float32)
        query_features = query_features / np.linalg.norm(query_features)
        
        # 搜索
        distances, indices = self.index.search(query_features.reshape(1, -1), top_k)
        
        # 获取结果
        results = []
        for idx, distance in zip(indices[0], distances[0]):
            if idx < len(self.image_paths):
                results.append((self.image_paths[idx], float(distance)))
        
        return results

# 使用示例
fbir = FeatureBasedImageRetrieval()
# features = np.random.rand(1000, 2048)
# image_paths = [f'/tmp/image_{i}.jpg' for i in range(1000)]
# fbir.build_index(features, image_paths)
# results = fbir.search(np.random.rand(2048), top_k=5)
​

3.3 音视频处理与分析

3.3.1 音频处理技术

音频基础预处理

import librosa
import numpy as np
from typing import Tuple, Optional
import soundfile as sf

class AudioPreprocessor:
    def __init__(self, sample_rate: int = 22050):
        self.sample_rate = sample_rate
    
    def load_audio(self, audio_path: str) -> Tuple[np.ndarray, int]:
        """加载音频文件"""
        audio, sr = librosa.load(audio_path, sr=self.sample_rate)
        return audio, sr
    
    def normalize_audio(self, audio: np.ndarray) -> np.ndarray:
        """音频归一化"""
        max_val = np.max(np.abs(audio))
        if max_val > 0:
            audio = audio / max_val
        return audio
    
    def extract_mfcc(self, audio: np.ndarray) -> np.ndarray:
        """提取MFCC特征"""
        mfcc = librosa.feature.mfcc(y=audio, sr=self.sample_rate, n_mfcc=13)
        return mfcc

# 使用示例
preprocessor = AudioPreprocessor()
# audio, sr = preprocessor.load_audio('/tmp/audio.wav')
# normalized = preprocessor.normalize_audio(audio)
# mfcc = preprocessor.extract_mfcc(normalized)
​

3.4 跨模态对齐与融合

3.4.1 跨模态对齐技术

基于注意力机制的对齐：

import torch
import torch.nn as nn
import numpy as np
from typing import Tuple

class CrossModalAttention(nn.Module):
    def __init__(self, text_dim: int, image_dim: int, hidden_dim: int = 256):
        super(CrossModalAttention, self).__init__()
        self.text_dim = text_dim
        self.image_dim = image_dim
        self.hidden_dim = hidden_dim
        
        # 注意力权重计算
        self.text_attention = nn.Linear(text_dim, hidden_dim)
        self.image_attention = nn.Linear(image_dim, hidden_dim)
        self.attention_combine = nn.Linear(hidden_dim * 2, 1)
        
        # 对齐层
        self.align_layer = nn.Linear(text_dim + image_dim, text_dim)
        
    def forward(self, text_features: torch.Tensor, 
                image_features: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
        batch_size, seq_len, _ = text_features.shape
        
        # 计算注意力权重
        text_att = self.text_attention(text_features)
        image_att = self.image_attention(image_features.unsqueeze(1))
        image_att = image_att.expand(-1, seq_len, -1)
        
        # 计算联合注意力
        combined = torch.cat([text_att, image_att], dim=-1)
        attention_scores = torch.softmax(self.attention_combine(combined), dim=1)
        
        # 应用注意力权重
        attended_text = torch.sum(attention_scores * text_features, dim=1)
        
        # 对齐操作
        aligned_features = torch.cat([attended_text, image_features], dim=-1)
        aligned_text = self.align_layer(aligned_features)
        
        return aligned_text, image_features

# 使用示例
cross_modal_attention = CrossModalAttention(
    text_dim=768, 
    image_dim=2048, 
    hidden_dim=256
)
​

3.4.2 跨模态融合技术

早期融合策略：

import torch
import torch.nn as nn

class EarlyFusion(nn.Module):
    def __init__(self, text_dim: int, image_dim: int, 
                 fusion_dim: int = 512, num_classes: int = 10):
        super(EarlyFusion, self).__init__()
        self.text_dim = text_dim
        self.image_dim = image_dim
        self.fusion_dim = fusion_dim
        
        # 特征提取层
        self.text_encoder = nn.Sequential(
            nn.Linear(text_dim, fusion_dim),
            nn.ReLU(),
            nn.Dropout(0.1)
        )
        
        self.image_encoder = nn.Sequential(
            nn.Linear(image_dim, fusion_dim),
            nn.ReLU(),
            nn.Dropout(0.1)
        )
        
        # 融合层
        self.fusion_layer = nn.Sequential(
            nn.Linear(fusion_dim * 2, fusion_dim),
            nn.ReLU(),
            nn.Dropout(0.1)
        )
        
        # 分类层
        self.classifier = nn.Linear(fusion_dim, num_classes)
        
    def forward(self, text_features: torch.Tensor, 
                image_features: torch.Tensor) -> torch.Tensor:
        # 分别提取特征
        text_encoded = self.text_encoder(text_features)
        image_encoded = self.image_encoder(image_features)
        
        # 早期融合
        fused = torch.cat([text_encoded, image_encoded], dim=-1)
        fused = self.fusion_layer(fused)
        
        # 分类
        logits = self.classifier(fused)
        
        return logits

# 使用示例
early_fusion = EarlyFusion(
    text_dim=768,
    image_dim=2048,
    fusion_dim=512,
    num_classes=10
)
​

本章技术要点总结

1. 图像检索技术：

   • 基于FAISS的相似性搜索
   • 特征向量化与索引构建
   • 高维空间中的距离计算

2. 音视频处理：

   • 音频预处理：加载、归一化、特征提取
   • MFCC特征用于音频识别
   • 时频分析在音视频处理中的应用

3. 跨模态对齐与融合：

   • 注意力机制实现模态间对齐
   • 早期融合与晚期融合策略
   • 深度学习模型的多模态整合

这些技术为构建高性能的多模态知识库系统提供了重要的技术支撑。