hubert-base-ser

<!-- This model card has been generated automatically according to the information the Trainer had access to. You should probably proofread and complete it, then remove this comment. -->

hubert-base-ser

This model is a fine-tuned version of TencentGameMate/chinese-hubert-base on an unknown dataset. It achieves the following results on the evaluation set:

• Loss: 0.1466
• Accuracy: 0.9526

How to use

Requirements

# requirement packages
!pip install git+https://github.com/huggingface/datasets.git
!pip install git+https://github.com/huggingface/transformers.git
!pip install torchaudio

Prediction

import os
import torch
import torchaudio
import torch.nn as nn
import torch.nn.functional as F
from typing import Optional, Tuple
from dataclasses import dataclass
from transformers import AutoConfig, Wav2Vec2FeatureExtractor, HubertPreTrainedModel, HubertModel
from transformers.file_utils import ModelOutput

def speech_file_to_array_fn(path, sampling_rate):
    speech_array, _sampling_rate = torchaudio.load(path)
    resampler = torchaudio.transforms.Resample(_sampling_rate,sampling_rate)
    speech = resampler(speech_array).squeeze().numpy()
    return speech


@dataclass
class SpeechClassifierOutput(ModelOutput):
    loss: Optional[torch.FloatTensor] = None
    logits: torch.FloatTensor = None
    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    attentions: Optional[Tuple[torch.FloatTensor]] = None

class HubertClassificationHead(nn.Module):
    """Head for hubert classification task."""

    def __init__(self, config):
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.dropout = nn.Dropout(config.final_dropout)
        self.out_proj = nn.Linear(config.hidden_size, config.num_labels)

    def forward(self, features, **kwargs):
        x = features
        x = self.dropout(x)
        x = self.dense(x)
        x = torch.tanh(x)
        x = self.dropout(x)
        x = self.out_proj(x)
        return x


class HubertForSpeechClassification(HubertPreTrainedModel):
    def __init__(self, config):
        super().__init__(config)
        self.config = config
        self.pooling_mode = config.pooling_mode

        self.hubert = HubertModel(config)
        self.classifier = HubertClassificationHead(config)
        self.init_weights()

    def merged_strategy(
            self,
            hidden_states,
            mode="mean"
    ):
        if mode == "mean":
            outputs = torch.mean(hidden_states, dim=1)
        elif mode == "sum":
            outputs = torch.sum(hidden_states, dim=1)
        elif mode == "max":
            outputs = torch.max(hidden_states, dim=1)[0]
        else:
            raise Exception(
                "The pooling method hasn't been defined! Your pooling mode must be one of these ['mean', 'sum', 'max']")

        return outputs

    def forward(self, x):
        outputs = self.hubert(x)
        hidden_states = outputs[0]
        hidden_states = self.merged_strategy(hidden_states, mode=self.pooling_mode)
        logits = self.classifier(hidden_states)
        # 返回SpeechClassifierOutput对象
        return SpeechClassifierOutput(logits=logits)


def main():
    print("正在加载模型...")
    
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model_name_or_path = "ZipperDeng/hubert-base-ser"
    config = AutoConfig.from_pretrained(model_name_or_path)
    feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model_name_or_path)
    sampling_rate = feature_extractor.sampling_rate
    model = HubertForSpeechClassification.from_pretrained(model_name_or_path).to(device)

    
    def predict_single_file(file_path, sampling_rate):
        """预测单个音频文件的情感"""
        try:
            speech = speech_file_to_array_fn(file_path, sampling_rate)
            features = feature_extractor(speech, sampling_rate=sampling_rate, return_tensors="pt", padding=True)

            input_values = features.input_values.to(device)

            with torch.no_grad():
                logits = model(input_values).logits

            scores = F.softmax(logits, dim=1).detach().cpu().numpy()[0]
            outputs = [{"Label": config.id2label[i], "Score": f"{round(score * 100, 3):.1f}%"} for i, score in enumerate(scores)]
            return outputs
        except Exception as e:
            print(f"处理文件 {file_path} 时出错: {e}")
            return None
    
    # 检查测试数据目录是否存在
    test_data = r"F:\test_ser"
    if not os.path.exists(test_data):
        print(f"测试数据目录不存在: {test_data}")
        print("请确保目录存在并包含音频文件")
        return
    
    file_path_list = [f"{test_data}/{path}" for path in os.listdir(f"{test_data}") if path.endswith(('.wav', '.mp3', '.flac'))]
    print(f"找到 {len(file_path_list)} 个音频文件")
    
    # 逐个处理每个文件
    for file_path in file_path_list:
        print(f"\n处理文件: {file_path}")
        outputs = predict_single_file(file_path, sampling_rate)
        print("预测结果:")
        for result in outputs:
            print(f"  {result['Label']}: {result['Score']}")


if __name__ == "__main__":
    # multiprocessing.freeze_support()
    main()

Model description

More information needed

Intended uses & limitations

More information needed

Training and evaluation data

More information needed

Training procedure

Training hyperparameters

The following hyperparameters were used during training:

• learning_rate: 0.0001
• train_batch_size: 32
• eval_batch_size: 4
• seed: 42
• gradient_accumulation_steps: 2
• total_train_batch_size: 64
• optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
• lr_scheduler_type: linear
• num_epochs: 1.0
• mixed_precision_training: Native AMP

Training results

Framework versions

• Transformers 4.47.0
• Pytorch 2.4.1+cu118
• Datasets 3.6.0
• Tokenizers 0.21.0