Add VibeVoice ASR (#48)
This commit is contained in:
mrfakename
@@ -12,6 +12,12 @@ from .configuration_vibevoice import (
|
||||
VibeVoiceAcousticTokenizerConfig,
|
||||
VibeVoiceSemanticTokenizerConfig,
|
||||
VibeVoiceDiffusionHeadConfig,
|
||||
VibeVoiceASRConfig,
|
||||
)
|
||||
from .modeling_vibevoice_asr import (
|
||||
VibeVoiceASRPreTrainedModel,
|
||||
VibeVoiceASRModel,
|
||||
VibeVoiceASRForConditionalGeneration,
|
||||
)
|
||||
from .modeling_vibevoice import (
|
||||
VibeVoicePreTrainedModel,
|
||||
@@ -67,10 +73,15 @@ __all__ = [
|
||||
"VibeVoiceAcousticTokenizerConfig",
|
||||
"VibeVoiceSemanticTokenizerConfig",
|
||||
"VibeVoiceDiffusionHeadConfig",
|
||||
"VibeVoiceASRConfig",
|
||||
# Multi-speaker models
|
||||
"VibeVoicePreTrainedModel",
|
||||
"VibeVoiceModel",
|
||||
"VibeVoiceForConditionalGenerationInference",
|
||||
# ASR models
|
||||
"VibeVoiceASRPreTrainedModel",
|
||||
"VibeVoiceASRModel",
|
||||
"VibeVoiceASRForConditionalGeneration",
|
||||
# Streaming configs
|
||||
"VibeVoiceStreamingConfig",
|
||||
# Streaming models
|
||||
|
||||
@@ -240,9 +240,112 @@ class VibeVoiceConfig(PretrainedConfig):
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
class VibeVoiceASRConfig(PretrainedConfig):
|
||||
model_type = "vibevoice"
|
||||
is_composition = True
|
||||
sub_configs = {
|
||||
"acoustic_tokenizer_config": VibeVoiceAcousticTokenizerConfig,
|
||||
"semantic_tokenizer_config": VibeVoiceSemanticTokenizerConfig,
|
||||
"decoder_config": Qwen2Config,
|
||||
}
|
||||
# keys_to_ignore_at_inference = ["past_key_values"]
|
||||
# Default tensor parallel plan for base model `Qwen2`
|
||||
base_model_tp_plan = {
|
||||
"layers.*.self_attn.q_proj": "colwise",
|
||||
"layers.*.self_attn.k_proj": "colwise",
|
||||
"layers.*.self_attn.v_proj": "colwise",
|
||||
"layers.*.self_attn.o_proj": "rowwise",
|
||||
"layers.*.mlp.gate_proj": "colwise",
|
||||
"layers.*.mlp.up_proj": "colwise",
|
||||
"layers.*.mlp.down_proj": "rowwise",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
acoustic_tokenizer_config=None,
|
||||
semantic_tokenizer_config=None,
|
||||
decoder_config=None,
|
||||
**kwargs
|
||||
):
|
||||
|
||||
# kwargs["_attn_implementation"] = "flash_attention_2"
|
||||
kwargs["_attn_implementation_autoset"] = False
|
||||
|
||||
if acoustic_tokenizer_config is None:
|
||||
self.acoustic_tokenizer_config = self.sub_configs["acoustic_tokenizer_config"]()
|
||||
elif isinstance(acoustic_tokenizer_config, dict):
|
||||
acoustic_tokenizer_config["model_type"] = "vibevoice_acoustic_tokenizer"
|
||||
self.acoustic_tokenizer_config = self.sub_configs["acoustic_tokenizer_config"](**acoustic_tokenizer_config)
|
||||
elif isinstance(acoustic_tokenizer_config, VibeVoiceAcousticTokenizerConfig):
|
||||
# If an instance of the config class is provided
|
||||
self.acoustic_tokenizer_config = acoustic_tokenizer_config
|
||||
|
||||
if semantic_tokenizer_config is None:
|
||||
self.semantic_tokenizer_config = self.sub_configs["semantic_tokenizer_config"]()
|
||||
elif isinstance(semantic_tokenizer_config, dict):
|
||||
semantic_tokenizer_config["model_type"] = "vibevoice_semantic_tokenizer"
|
||||
self.semantic_tokenizer_config = self.sub_configs["semantic_tokenizer_config"](**semantic_tokenizer_config)
|
||||
elif isinstance(semantic_tokenizer_config, VibeVoiceSemanticTokenizerConfig):
|
||||
# If an instance of the config class is provided
|
||||
self.semantic_tokenizer_config = semantic_tokenizer_config
|
||||
|
||||
if decoder_config is None:
|
||||
self.decoder_config = self.sub_configs["decoder_config"]()
|
||||
elif isinstance(decoder_config, dict):
|
||||
# If a dictionary is provided, instantiate the config class with it
|
||||
# self.decoder_config = self.sub_configs["decoder_config"](**decoder_config)
|
||||
if decoder_config.get("model_type", '') == "qwen2":
|
||||
self.decoder_config = Qwen2Config(**decoder_config)
|
||||
else:
|
||||
raise ValueError(f"Unsupported decoder model type: {decoder_config.get('model_type', '')}")
|
||||
elif isinstance(decoder_config, Qwen2Config):
|
||||
# If an instance of the config class is provided
|
||||
self.decoder_config = decoder_config
|
||||
|
||||
# other parameters
|
||||
self.acoustic_vae_dim = getattr(self.acoustic_tokenizer_config, 'vae_dim', 64)
|
||||
self.semantic_vae_dim = getattr(self.semantic_tokenizer_config, 'vae_dim', 128)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def get_text_config(self, decoder: bool = False):
|
||||
"""Return the text (decoder) config for generation."""
|
||||
return self.decoder_config
|
||||
|
||||
@property
|
||||
def vocab_size(self):
|
||||
"""Return vocab_size from decoder config for generation compatibility."""
|
||||
return self.decoder_config.vocab_size
|
||||
|
||||
@property
|
||||
def num_attention_heads(self):
|
||||
"""Return num_attention_heads from decoder config for Ulysses SP compatibility."""
|
||||
return self.decoder_config.num_attention_heads
|
||||
|
||||
@property
|
||||
def num_key_value_heads(self):
|
||||
"""Return num_key_value_heads from decoder config for Ulysses SP compatibility."""
|
||||
return self.decoder_config.num_key_value_heads
|
||||
|
||||
@property
|
||||
def hidden_size(self):
|
||||
"""Return hidden_size from decoder config for model compatibility."""
|
||||
return self.decoder_config.hidden_size
|
||||
|
||||
@property
|
||||
def num_hidden_layers(self):
|
||||
"""Return num_hidden_layers from decoder config for Ulysses SP compatibility."""
|
||||
return self.decoder_config.num_hidden_layers
|
||||
|
||||
@property
|
||||
def head_dim(self):
|
||||
"""Return head_dim from decoder config for Ulysses SP compatibility."""
|
||||
return getattr(self.decoder_config, 'head_dim', self.hidden_size // self.num_attention_heads)
|
||||
|
||||
__all__ = [
|
||||
"VibeVoiceAcousticTokenizerConfig",
|
||||
"VibeVoiceSemanticTokenizerConfig",
|
||||
"VibeVoiceDiffusionHeadConfig",
|
||||
"VibeVoiceConfig"
|
||||
"VibeVoiceAcousticTokenizerConfig",
|
||||
"VibeVoiceSemanticTokenizerConfig",
|
||||
"VibeVoiceDiffusionHeadConfig",
|
||||
"VibeVoiceConfig",
|
||||
"VibeVoiceASRConfig"
|
||||
]
|
||||
@@ -0,0 +1,520 @@
|
||||
from typing import List, Optional, Tuple, Union
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from transformers.models.auto import AutoModel, AutoModelForCausalLM
|
||||
|
||||
from transformers.modeling_outputs import CausalLMOutput, BaseModelOutputWithPast
|
||||
from transformers import modeling_utils
|
||||
from transformers.modeling_utils import PreTrainedModel
|
||||
from transformers.utils import logging
|
||||
from transformers.generation import GenerationMixin
|
||||
|
||||
from .modular_vibevoice_tokenizer import (
|
||||
VibeVoiceTokenizerStreamingCache,
|
||||
VibeVoiceTokenizerEncoderOutput
|
||||
)
|
||||
|
||||
from .configuration_vibevoice import VibeVoiceASRConfig
|
||||
from .modeling_vibevoice import (
|
||||
VibeVoiceCausalLMOutputWithPast,
|
||||
SpeechConnector
|
||||
)
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
if not hasattr(modeling_utils, "ALL_PARALLEL_STYLES") or modeling_utils.ALL_PARALLEL_STYLES is None:
|
||||
modeling_utils.ALL_PARALLEL_STYLES = ["tp", "none", "colwise", "rowwise"]
|
||||
|
||||
# @auto_docstring
|
||||
class VibeVoiceASRPreTrainedModel(PreTrainedModel):
|
||||
config_class = VibeVoiceASRConfig
|
||||
base_model_prefix = "model"
|
||||
supports_gradient_checkpointing = True
|
||||
_skip_keys_device_placement = "past_key_values"
|
||||
_supports_cache_class = True
|
||||
_supports_flash_attn = True
|
||||
_supports_flash_attn_2 = True
|
||||
_supports_sdpa = True
|
||||
_supports_quantized_cache = True
|
||||
_supports_static_cache = True
|
||||
_supports_attention_backend = True
|
||||
|
||||
def _init_weights(self, module):
|
||||
|
||||
# Use the language model's initializer_range if available
|
||||
if hasattr(self.config, 'language_model_config') and hasattr(self.config.language_model_config, 'initializer_range'):
|
||||
std = self.config.language_model_config.initializer_range
|
||||
elif hasattr(self.config, 'decoder_config') and hasattr(self.config.decoder_config, 'initializer_range'):
|
||||
std = self.config.decoder_config.initializer_range
|
||||
else:
|
||||
std = 0.02 # Default value
|
||||
|
||||
if isinstance(module, nn.Linear):
|
||||
module.weight.data.normal_(mean=0.0, std=std)
|
||||
if module.bias is not None:
|
||||
module.bias.data.zero_()
|
||||
elif isinstance(module, nn.LayerNorm):
|
||||
module.weight.data.fill_(1.0)
|
||||
module.bias.data.zero_()
|
||||
|
||||
# @auto_docstring
|
||||
class VibeVoiceASRModel(VibeVoiceASRPreTrainedModel):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
|
||||
if hasattr(config, 'torch_dtype') and config.torch_dtype is not None:
|
||||
if isinstance(config.torch_dtype, str):
|
||||
dtype = getattr(torch, config.torch_dtype)
|
||||
else:
|
||||
dtype = config.torch_dtype
|
||||
else:
|
||||
dtype = torch.float32
|
||||
|
||||
# Initialize Qwen2 model for language modeling
|
||||
lm_config = config.decoder_config
|
||||
self.language_model = AutoModel.from_config(lm_config)
|
||||
|
||||
# Initialize speech components if needed
|
||||
self.acoustic_tokenizer = AutoModel.from_config(config.acoustic_tokenizer_config).to(dtype)
|
||||
self.semantic_tokenizer = AutoModel.from_config(config.semantic_tokenizer_config).to(dtype)
|
||||
|
||||
self.acoustic_connector = SpeechConnector(config.acoustic_vae_dim, lm_config.hidden_size).to(dtype)
|
||||
self.semantic_connector = SpeechConnector(config.semantic_vae_dim, lm_config.hidden_size).to(dtype)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
if hasattr(self.language_model, 'embed_tokens'):
|
||||
# If the language model has an embed_tokens attribute, return it
|
||||
return self.language_model.embed_tokens
|
||||
|
||||
for name, attr in self.language_model.fullmap.items(): # parallel by nnscaler, the name is changed
|
||||
if attr.orig_name == 'embed_tokens.weight':
|
||||
return getattr(self.language_model, name)
|
||||
assert False, 'should not arrive here'
|
||||
|
||||
def set_input_embeddings(self, value):
|
||||
self.language_model.embed_tokens = value
|
||||
|
||||
def set_speech_tokenizers(self, acoustic_tokenizer=None, semantic_tokenizer=None):
|
||||
"""Set the speech tokenizers used for encoding and decoding speech."""
|
||||
self.acoustic_tokenizer = acoustic_tokenizer
|
||||
self.semantic_tokenizer = semantic_tokenizer
|
||||
|
||||
# Reset the encoder to evaluation mode
|
||||
if self.acoustic_tokenizer is not None:
|
||||
self.acoustic_tokenizer.eval()
|
||||
|
||||
if self.semantic_tokenizer is not None:
|
||||
self.semantic_tokenizer.eval()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.LongTensor = None,
|
||||
attention_mask: Optional[torch.Tensor] = None,
|
||||
position_ids: Optional[torch.LongTensor] = None,
|
||||
past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
|
||||
inputs_embeds: Optional[torch.FloatTensor] = None,
|
||||
use_cache: Optional[bool] = None,
|
||||
output_attentions: Optional[bool] = None,
|
||||
output_hidden_states: Optional[bool] = None,
|
||||
return_dict: Optional[bool] = None,
|
||||
cache_position: Optional[torch.LongTensor] = None,
|
||||
**kwargs,
|
||||
) -> Union[Tuple, BaseModelOutputWithPast]:
|
||||
|
||||
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
||||
|
||||
# Forward through language model
|
||||
outputs = self.language_model(
|
||||
input_ids=input_ids,
|
||||
attention_mask=attention_mask,
|
||||
position_ids=position_ids,
|
||||
past_key_values=past_key_values,
|
||||
inputs_embeds=inputs_embeds,
|
||||
use_cache=use_cache,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
cache_position=cache_position,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if not return_dict:
|
||||
return outputs
|
||||
|
||||
return BaseModelOutputWithPast(
|
||||
last_hidden_state=outputs.last_hidden_state,
|
||||
past_key_values=outputs.past_key_values,
|
||||
hidden_states=outputs.hidden_states,
|
||||
attentions=outputs.attentions,
|
||||
)
|
||||
|
||||
class VibeVoiceASRForConditionalGeneration(VibeVoiceASRPreTrainedModel, GenerationMixin):
|
||||
"""
|
||||
VibeVoice model for Automatic Speech Recognition (ASR) with language modeling head for conditional generation.
|
||||
This class is designed for inference and generation tasks.
|
||||
"""
|
||||
_tied_weights_keys = ["lm_head.weight"]
|
||||
_tp_plan = {"lm_head": "colwise_rep"}
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.model = VibeVoiceASRModel(config)
|
||||
self.vocab_size = config.decoder_config.vocab_size
|
||||
|
||||
# Determine the dtype to use
|
||||
if hasattr(config, 'torch_dtype') and config.torch_dtype is not None:
|
||||
if isinstance(config.torch_dtype, str):
|
||||
dtype = getattr(torch, config.torch_dtype)
|
||||
else:
|
||||
dtype = config.torch_dtype
|
||||
else:
|
||||
dtype = torch.float32
|
||||
|
||||
# Initialize lm_head with the correct dtype
|
||||
self.lm_head = nn.Linear(config.decoder_config.hidden_size, self.vocab_size, bias=False).to(dtype)
|
||||
|
||||
# Initialize weights and apply final processing
|
||||
self.post_init()
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.model.get_input_embeddings()
|
||||
|
||||
def set_input_embeddings(self, value):
|
||||
self.model.set_input_embeddings(value)
|
||||
|
||||
def get_output_embeddings(self):
|
||||
return self.lm_head
|
||||
|
||||
def set_output_embeddings(self, new_embeddings):
|
||||
self.lm_head = new_embeddings
|
||||
|
||||
def set_decoder(self, decoder):
|
||||
self.model.language_model = decoder
|
||||
|
||||
def get_decoder(self):
|
||||
return self.model.language_model
|
||||
|
||||
def tie_weights(self):
|
||||
"""Tie the weights between the input embeddings and the output embeddings."""
|
||||
if getattr(self.config.decoder_config, 'tie_word_embeddings', False):
|
||||
output_embeddings = self.get_output_embeddings()
|
||||
input_embeddings = self.get_input_embeddings()
|
||||
if hasattr(input_embeddings, 'weight'):
|
||||
output_embeddings.weight = input_embeddings.weight
|
||||
else:
|
||||
output_embeddings.weight = input_embeddings
|
||||
|
||||
def encode_speech(
|
||||
self,
|
||||
speech_tensors: torch.FloatTensor,
|
||||
speech_masks: Optional[torch.BoolTensor] = None,
|
||||
speech_semantic_tensors: Optional[torch.FloatTensor] = None,
|
||||
streaming_segment_duration: float = 60.0, # seconds
|
||||
):
|
||||
"""
|
||||
Encode speech input into features that can be used by the language model.
|
||||
This method is called once before generation to process the speech input.
|
||||
|
||||
For long audio (>600s by default), uses streaming processing to avoid conv overflow (>2^32).
|
||||
Segments are processed independently, then concatenated before final sampling.
|
||||
|
||||
Args:
|
||||
speech_tensors: Input audio tensor [batch_size, samples]
|
||||
speech_masks: Optional mask for speech features
|
||||
speech_semantic_tensors: Optional pre-computed semantic tokens
|
||||
streaming_segment_duration: Segment duration in seconds for streaming processing (default: 60s)
|
||||
"""
|
||||
if hasattr(self.config, 'torch_dtype') and self.config.torch_dtype is not None:
|
||||
if isinstance(self.config.torch_dtype, str):
|
||||
dtype = getattr(torch, self.config.torch_dtype)
|
||||
else:
|
||||
dtype = self.config.torch_dtype
|
||||
else:
|
||||
dtype = torch.float32
|
||||
|
||||
speech_tensors = speech_tensors.to(dtype)
|
||||
|
||||
# Ensure proper shape: (batch, samples)
|
||||
if speech_tensors.ndim == 1:
|
||||
speech_tensors = speech_tensors.unsqueeze(0)
|
||||
|
||||
batch_size, total_samples = speech_tensors.shape
|
||||
sample_rate = 24000 # fix 24kHz sample rate
|
||||
|
||||
# Calculate segment size in samples
|
||||
segment_samples = int(streaming_segment_duration * sample_rate)
|
||||
|
||||
# Decide whether to use streaming based on audio length
|
||||
use_streaming = total_samples > segment_samples
|
||||
|
||||
with torch.no_grad():
|
||||
if not use_streaming:
|
||||
# Short audio: direct processing (original behavior)
|
||||
encoder_output = self.model.acoustic_tokenizer.encode(speech_tensors.unsqueeze(1))
|
||||
audio_tokens = encoder_output.sample(dist_type=self.model.acoustic_tokenizer.std_dist_type)[0]
|
||||
acoustic_features = self.model.acoustic_connector(audio_tokens)
|
||||
|
||||
# Encode semantic features
|
||||
if speech_semantic_tensors is not None:
|
||||
semantic_features = self.model.semantic_connector(speech_semantic_tensors)
|
||||
else:
|
||||
semantic_tokens = self.model.semantic_tokenizer.encode(speech_tensors.unsqueeze(1)).mean
|
||||
semantic_features = self.model.semantic_connector(semantic_tokens)
|
||||
else:
|
||||
# Long audio: streaming processing
|
||||
# print(f"Using streaming processing for long audio: {total_samples/sample_rate:.1f}s "
|
||||
# f"(segment size: {streaming_segment_duration}s)")
|
||||
|
||||
# Initialize caches for both tokenizers
|
||||
acoustic_encoder_cache = VibeVoiceTokenizerStreamingCache()
|
||||
semantic_encoder_cache = VibeVoiceTokenizerStreamingCache()
|
||||
acoustic_mean_segments = []
|
||||
semantic_mean_segments = []
|
||||
sample_indices = torch.arange(batch_size, device=speech_tensors.device)
|
||||
|
||||
# Helper function from batch_asr_sft_cache.py
|
||||
def _iter_segments(total_length: int, segment_length: int):
|
||||
"""Iterate over audio segments with a given segment length."""
|
||||
if segment_length <= 0:
|
||||
raise ValueError("segment_length must be positive")
|
||||
for start in range(0, total_length, segment_length):
|
||||
end = min(start + segment_length, total_length)
|
||||
if end > start:
|
||||
yield start, end
|
||||
|
||||
# Process each segment for both acoustic and semantic tokenizers
|
||||
segments = list(_iter_segments(total_samples, segment_samples))
|
||||
num_segments = len(segments)
|
||||
for seg_idx, (start, end) in enumerate(segments):
|
||||
chunk = speech_tensors[:, start:end].contiguous()
|
||||
if chunk.numel() == 0:
|
||||
continue
|
||||
|
||||
# Check if this is the final segment
|
||||
is_final = (seg_idx == num_segments - 1)
|
||||
|
||||
# Encode chunk for acoustic tokenizer (don't sample yet)
|
||||
acoustic_encoder_output = self.model.acoustic_tokenizer.encode(
|
||||
chunk.unsqueeze(1),
|
||||
cache=acoustic_encoder_cache,
|
||||
sample_indices=sample_indices,
|
||||
use_cache=True,
|
||||
is_final_chunk=is_final,
|
||||
)
|
||||
acoustic_mean_segments.append(acoustic_encoder_output.mean)
|
||||
|
||||
# Encode chunk for semantic tokenizer (take mean directly)
|
||||
semantic_encoder_output = self.model.semantic_tokenizer.encode(
|
||||
chunk.unsqueeze(1),
|
||||
cache=semantic_encoder_cache,
|
||||
sample_indices=sample_indices,
|
||||
use_cache=True,
|
||||
is_final_chunk=is_final,
|
||||
)
|
||||
semantic_mean_segments.append(semantic_encoder_output.mean)
|
||||
|
||||
# print(f"Processed {len(acoustic_mean_segments)} segments.")
|
||||
# Concatenate all acoustic means and sample once
|
||||
acoustic_mean_full = torch.cat(acoustic_mean_segments, dim=1).contiguous()
|
||||
acoustic_encoder_output = VibeVoiceTokenizerEncoderOutput(
|
||||
mean=acoustic_mean_full,
|
||||
std=self.model.acoustic_tokenizer.fix_std
|
||||
)
|
||||
audio_tokens = acoustic_encoder_output.sample(
|
||||
dist_type=self.model.acoustic_tokenizer.std_dist_type
|
||||
)[0]
|
||||
acoustic_features = self.model.acoustic_connector(audio_tokens)
|
||||
|
||||
# Concatenate all semantic means
|
||||
semantic_tokens = torch.cat(semantic_mean_segments, dim=1).contiguous()
|
||||
semantic_features = self.model.semantic_connector(semantic_tokens)
|
||||
|
||||
# Combine acoustic and semantic features
|
||||
if speech_masks is not None:
|
||||
combined_features = acoustic_features[speech_masks] + semantic_features[speech_masks]
|
||||
else:
|
||||
combined_features = acoustic_features + semantic_features
|
||||
|
||||
return combined_features
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: Optional[torch.LongTensor] = None,
|
||||
attention_mask: Optional[torch.Tensor] = None,
|
||||
position_ids: Optional[torch.LongTensor] = None,
|
||||
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
||||
inputs_embeds: Optional[torch.FloatTensor] = None,
|
||||
labels: Optional[torch.LongTensor] = None,
|
||||
use_cache: Optional[bool] = None,
|
||||
output_attentions: Optional[bool] = None,
|
||||
output_hidden_states: Optional[bool] = None,
|
||||
return_dict: Optional[bool] = None,
|
||||
cache_position: Optional[torch.LongTensor] = None,
|
||||
# Speech-specific arguments
|
||||
speech_tensors: Optional[torch.FloatTensor] = None,
|
||||
speech_masks: Optional[torch.BoolTensor] = None,
|
||||
speech_semantic_tensors: Optional[torch.FloatTensor] = None,
|
||||
acoustic_input_mask: Optional[torch.BoolTensor] = None,
|
||||
**kwargs,
|
||||
) -> Union[Tuple, CausalLMOutput]:
|
||||
"""
|
||||
Forward pass for the model. Handles both training and generation scenarios.
|
||||
"""
|
||||
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
||||
output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
||||
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
||||
use_cache = use_cache if use_cache is not None else self.config.use_cache
|
||||
|
||||
# Process inputs
|
||||
if inputs_embeds is None and input_ids is not None:
|
||||
inputs_embeds = self.get_input_embeddings()(input_ids)
|
||||
|
||||
# If we have speech input and acoustic_input_mask, encode and insert speech features
|
||||
if speech_tensors is not None and acoustic_input_mask is not None:
|
||||
speech_features = self.encode_speech(
|
||||
speech_tensors=speech_tensors,
|
||||
speech_masks=speech_masks,
|
||||
speech_semantic_tensors=speech_semantic_tensors,
|
||||
)
|
||||
inputs_embeds[acoustic_input_mask] = speech_features
|
||||
|
||||
# Forward through the model
|
||||
outputs = self.model(
|
||||
input_ids=None,
|
||||
attention_mask=attention_mask,
|
||||
position_ids=position_ids,
|
||||
past_key_values=past_key_values,
|
||||
inputs_embeds=inputs_embeds,
|
||||
use_cache=use_cache,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
cache_position=cache_position,
|
||||
)
|
||||
|
||||
hidden_states = outputs[0] if not return_dict else outputs.last_hidden_state
|
||||
logits = self.lm_head(hidden_states)
|
||||
|
||||
loss = None
|
||||
if labels is not None:
|
||||
# Shift so that tokens < n predict n
|
||||
shift_logits = logits[..., :-1, :].contiguous()
|
||||
shift_labels = labels[..., 1:].contiguous()
|
||||
# Flatten the tokens
|
||||
loss_fct = nn.CrossEntropyLoss()
|
||||
shift_logits = shift_logits.view(-1, self.vocab_size)
|
||||
shift_labels = shift_labels.view(-1)
|
||||
# Enable model parallelism
|
||||
shift_labels = shift_labels.to(shift_logits.device)
|
||||
loss = loss_fct(shift_logits, shift_labels)
|
||||
|
||||
if not return_dict:
|
||||
output = (logits,) + outputs[1:]
|
||||
return (loss,) + output if loss is not None else output
|
||||
|
||||
return VibeVoiceCausalLMOutputWithPast(
|
||||
loss=loss,
|
||||
logits=logits,
|
||||
past_key_values=outputs.past_key_values,
|
||||
hidden_states=outputs.hidden_states,
|
||||
attentions=outputs.attentions,
|
||||
)
|
||||
|
||||
def prepare_inputs_for_generation(
|
||||
self,
|
||||
input_ids,
|
||||
past_key_values=None,
|
||||
attention_mask=None,
|
||||
inputs_embeds=None,
|
||||
cache_position=None,
|
||||
position_ids=None,
|
||||
use_cache=True,
|
||||
speech_tensors=None,
|
||||
speech_masks=None,
|
||||
speech_semantic_tensors=None,
|
||||
acoustic_input_mask=None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Prepare inputs for generation step. This method is called by generate()
|
||||
for each token generation step.
|
||||
|
||||
Following Qwen2-VL's approach: speech inputs are only forwarded on the first pass
|
||||
(when cache_position[0] == 0), and are excluded in subsequent generation steps.
|
||||
"""
|
||||
# If we have past key values, we only need to process the new tokens
|
||||
if past_key_values is not None:
|
||||
if isinstance(past_key_values, tuple):
|
||||
past_length = past_key_values[0][0].shape[2]
|
||||
else:
|
||||
past_length = past_key_values.get_seq_length()
|
||||
|
||||
# Keep only the new tokens
|
||||
if input_ids is not None and input_ids.shape[1] > past_length:
|
||||
input_ids = input_ids[:, past_length:]
|
||||
|
||||
# Prepare position ids
|
||||
if position_ids is None and attention_mask is not None:
|
||||
position_ids = attention_mask.long().cumsum(-1) - 1
|
||||
position_ids.masked_fill_(attention_mask == 0, 1)
|
||||
if past_key_values is not None and input_ids is not None:
|
||||
position_ids = position_ids[:, -input_ids.shape[1]:]
|
||||
|
||||
# Prepare cache position
|
||||
if cache_position is None:
|
||||
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
|
||||
cache_position = torch.arange(
|
||||
past_seen_tokens,
|
||||
past_seen_tokens + (input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]),
|
||||
device=input_ids.device if input_ids is not None else inputs_embeds.device
|
||||
)
|
||||
|
||||
# Prepare model inputs
|
||||
if inputs_embeds is not None and past_key_values is None:
|
||||
model_inputs = {"inputs_embeds": inputs_embeds}
|
||||
else:
|
||||
model_inputs = {"input_ids": input_ids}
|
||||
|
||||
model_inputs.update(
|
||||
{
|
||||
"position_ids": position_ids,
|
||||
"cache_position": cache_position,
|
||||
"past_key_values": past_key_values,
|
||||
"use_cache": use_cache,
|
||||
"attention_mask": attention_mask,
|
||||
}
|
||||
)
|
||||
|
||||
# Following Qwen2-VL pattern: only include speech inputs on the first forward pass
|
||||
# (when cache_position[0] == 0), exclude them in subsequent generation steps
|
||||
if cache_position is not None and len(cache_position) > 0 and cache_position[0] == 0:
|
||||
# First forward pass - include speech inputs if provided
|
||||
model_inputs.update({
|
||||
"speech_tensors": speech_tensors,
|
||||
"speech_masks": speech_masks,
|
||||
"speech_semantic_tensors": speech_semantic_tensors,
|
||||
"acoustic_input_mask": acoustic_input_mask,
|
||||
})
|
||||
else:
|
||||
# Subsequent generation steps - exclude speech inputs
|
||||
model_inputs.update({
|
||||
"speech_tensors": None,
|
||||
"speech_masks": None,
|
||||
"speech_semantic_tensors": None,
|
||||
"acoustic_input_mask": None,
|
||||
})
|
||||
|
||||
# Include any remaining kwargs that might be needed
|
||||
model_inputs.update(kwargs)
|
||||
|
||||
return model_inputs
|
||||
|
||||
AutoModel.register(VibeVoiceASRConfig, VibeVoiceASRModel)
|
||||
AutoModelForCausalLM.register(VibeVoiceASRConfig, VibeVoiceASRForConditionalGeneration)
|
||||
|
||||
__all__ = [
|
||||
"VibeVoiceASRPreTrainedModel",
|
||||
"VibeVoiceASRModel",
|
||||
"VibeVoiceASRForConditionalGeneration",
|
||||
]
|
||||
@@ -9,10 +9,12 @@ This module provides processors for preparing inputs for VibeVoice models:
|
||||
from .vibevoice_processor import VibeVoiceProcessor
|
||||
from .vibevoice_streaming_processor import VibeVoiceStreamingProcessor
|
||||
from .vibevoice_tokenizer_processor import VibeVoiceTokenizerProcessor, AudioNormalizer
|
||||
from .vibevoice_asr_processor import VibeVoiceASRProcessor
|
||||
|
||||
__all__ = [
|
||||
"VibeVoiceProcessor",
|
||||
"VibeVoiceStreamingProcessor",
|
||||
"VibeVoiceTokenizerProcessor",
|
||||
"AudioNormalizer",
|
||||
"VibeVoiceASRProcessor",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,572 @@
|
||||
"""
|
||||
Processor class for VibeVoice ASR models.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import math
|
||||
import warnings
|
||||
from typing import List, Optional, Union, Dict, Any, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from transformers.tokenization_utils_base import BatchEncoding
|
||||
from transformers.utils import TensorType, logging
|
||||
from .vibevoice_tokenizer_processor import VibeVoiceTokenizerProcessor, AudioNormalizer
|
||||
|
||||
try:
|
||||
from .audio_utils import load_audio_use_ffmpeg
|
||||
HAS_FFMPEG_UTILS = True
|
||||
except ImportError:
|
||||
HAS_FFMPEG_UTILS = False
|
||||
warnings.warn("audio_utils not available, will fall back to soundfile for audio loading")
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
SYSTEM_PROMPT = "You are a helpful assistant that transcribes audio input into text output in JSON format."
|
||||
|
||||
|
||||
class VibeVoiceASRProcessor:
|
||||
"""
|
||||
Processor for VibeVoice ASR (Automatic Speech Recognition) models.
|
||||
|
||||
This processor handles audio preprocessing and tokenization for ASR tasks,
|
||||
following the exact format used in training with proper chat templates.
|
||||
|
||||
Args:
|
||||
tokenizer: The text tokenizer for processing text
|
||||
audio_processor: The audio processor for processing speech
|
||||
speech_tok_compress_ratio (int): Compression ratio for speech tokenization
|
||||
target_sample_rate (int): Target sample rate for audio
|
||||
normalize_audio (bool): Whether to normalize audio input
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer=None,
|
||||
audio_processor=None,
|
||||
speech_tok_compress_ratio=320,
|
||||
target_sample_rate=24000,
|
||||
normalize_audio=True,
|
||||
**kwargs
|
||||
):
|
||||
self.tokenizer = tokenizer
|
||||
self.audio_processor = audio_processor or VibeVoiceTokenizerProcessor(
|
||||
sampling_rate=target_sample_rate,
|
||||
normalize_audio=normalize_audio
|
||||
)
|
||||
self.speech_tok_compress_ratio = speech_tok_compress_ratio
|
||||
self.target_sample_rate = target_sample_rate
|
||||
self.normalize_audio = normalize_audio
|
||||
|
||||
if normalize_audio:
|
||||
self.audio_normalizer = AudioNormalizer()
|
||||
else:
|
||||
self.audio_normalizer = None
|
||||
|
||||
# Cache special token IDs
|
||||
self._cache_special_tokens()
|
||||
|
||||
def _cache_special_tokens(self):
|
||||
"""Cache special token IDs for efficiency."""
|
||||
# Add safety checks for special tokens
|
||||
if hasattr(self.tokenizer, 'speech_start_id'):
|
||||
self.speech_start_id = self.tokenizer.speech_start_id
|
||||
else:
|
||||
self.speech_start_id = self.tokenizer.convert_tokens_to_ids("<|speech_start|>")
|
||||
|
||||
if hasattr(self.tokenizer, 'speech_end_id'):
|
||||
self.speech_end_id = self.tokenizer.speech_end_id
|
||||
else:
|
||||
self.speech_end_id = self.tokenizer.convert_tokens_to_ids("<|speech_end|>")
|
||||
|
||||
if hasattr(self.tokenizer, 'speech_pad_id'):
|
||||
self.speech_pad_id = self.tokenizer.speech_pad_id
|
||||
else:
|
||||
self.speech_pad_id = self.tokenizer.convert_tokens_to_ids("<|speech_pad|>")
|
||||
|
||||
if hasattr(self.tokenizer, 'pad_id'):
|
||||
self.pad_id = self.tokenizer.pad_id
|
||||
elif hasattr(self.tokenizer, 'pad_token_id'):
|
||||
self.pad_id = self.tokenizer.pad_token_id
|
||||
else:
|
||||
self.pad_id = self.tokenizer.convert_tokens_to_ids("<|endoftext|>")
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
|
||||
"""
|
||||
Load processor from a pretrained model path.
|
||||
|
||||
Args:
|
||||
pretrained_model_name_or_path: Path to the pretrained model
|
||||
**kwargs: Additional keyword arguments
|
||||
|
||||
Returns:
|
||||
VibeVoiceASRProcessor: The loaded processor
|
||||
"""
|
||||
import json
|
||||
from transformers.utils import cached_file
|
||||
from vibevoice.modular.modular_vibevoice_text_tokenizer import VibeVoiceASRTextTokenizerFast
|
||||
|
||||
# Try to load configuration
|
||||
config_path = os.path.join(pretrained_model_name_or_path, "preprocessor_config.json")
|
||||
config = {}
|
||||
|
||||
if os.path.exists(config_path):
|
||||
with open(config_path, 'r') as f:
|
||||
config = json.load(f)
|
||||
else:
|
||||
try:
|
||||
config_file = cached_file(
|
||||
pretrained_model_name_or_path,
|
||||
"preprocessor_config.json",
|
||||
**kwargs
|
||||
)
|
||||
with open(config_file, 'r') as f:
|
||||
config = json.load(f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not load preprocessor_config.json: {e}")
|
||||
logger.warning("Using default configuration")
|
||||
|
||||
# Extract parameters
|
||||
speech_tok_compress_ratio = config.get("speech_tok_compress_ratio", 3200)
|
||||
target_sample_rate = config.get("target_sample_rate", 24000)
|
||||
normalize_audio = config.get("normalize_audio", True)
|
||||
|
||||
# Load tokenizer
|
||||
language_model_pretrained_name = config.get("language_model_pretrained_name", None) or kwargs.pop("language_model_pretrained_name", "Qwen/Qwen2.5-1.5B")
|
||||
logger.info(f"Loading tokenizer from {language_model_pretrained_name}")
|
||||
|
||||
if 'qwen' in language_model_pretrained_name.lower():
|
||||
tokenizer = VibeVoiceASRTextTokenizerFast.from_pretrained(
|
||||
language_model_pretrained_name,
|
||||
**kwargs
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported tokenizer type for {language_model_pretrained_name}")
|
||||
|
||||
# Load audio processor
|
||||
audio_processor = VibeVoiceTokenizerProcessor(
|
||||
sampling_rate=target_sample_rate,
|
||||
normalize_audio=normalize_audio,
|
||||
target_dB_FS=config.get("target_dB_FS", -25),
|
||||
eps=config.get("eps", 1e-6),
|
||||
)
|
||||
|
||||
return cls(
|
||||
tokenizer=tokenizer,
|
||||
audio_processor=audio_processor,
|
||||
speech_tok_compress_ratio=speech_tok_compress_ratio,
|
||||
target_sample_rate=target_sample_rate,
|
||||
normalize_audio=normalize_audio,
|
||||
)
|
||||
|
||||
def save_pretrained(self, save_directory: Union[str, os.PathLike], **kwargs):
|
||||
"""
|
||||
Save processor configuration to a directory.
|
||||
|
||||
Args:
|
||||
save_directory: Directory to save the configuration
|
||||
**kwargs: Additional keyword arguments
|
||||
"""
|
||||
import json
|
||||
|
||||
os.makedirs(save_directory, exist_ok=True)
|
||||
|
||||
# Save processor configuration
|
||||
processor_config = {
|
||||
"processor_class": "VibeVoiceASRProcessor",
|
||||
"speech_tok_compress_ratio": self.speech_tok_compress_ratio,
|
||||
"target_sample_rate": self.target_sample_rate,
|
||||
"normalize_audio": self.normalize_audio,
|
||||
"target_dB_FS": -25,
|
||||
"eps": 1e-6,
|
||||
}
|
||||
|
||||
config_path = os.path.join(save_directory, "preprocessor_config.json")
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump(processor_config, f, indent=2)
|
||||
|
||||
logger.info(f"Processor configuration saved in {config_path}")
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
audio: Optional[Union[str, np.ndarray, torch.Tensor, List[Union[str, np.ndarray, torch.Tensor]]]] = None,
|
||||
sampling_rate: Optional[int] = None,
|
||||
return_tensors: Optional[Union[str, TensorType]] = None,
|
||||
padding: bool = True,
|
||||
max_length: Optional[int] = None,
|
||||
truncation: bool = False,
|
||||
add_generation_prompt: bool = True,
|
||||
use_streaming: bool = True,
|
||||
context_info: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> BatchEncoding:
|
||||
"""
|
||||
Process audio input for ASR model.
|
||||
|
||||
Args:
|
||||
audio: Audio input(s). Can be:
|
||||
- str: Path to audio file
|
||||
- np.ndarray: Audio array
|
||||
- torch.Tensor: Audio tensor
|
||||
- List of the above for batch processing
|
||||
sampling_rate: Sampling rate of input audio
|
||||
return_tensors: Output format ('pt' for PyTorch, 'np' for NumPy)
|
||||
padding: Whether to pad batch inputs
|
||||
max_length: Maximum sequence length
|
||||
truncation: Whether to truncate long sequences
|
||||
add_generation_prompt: Whether to add generation prompt for inference
|
||||
use_streaming: Whether to use streaming mode (True by default, auto False if <60s)
|
||||
context_info: Optional context information (e.g., hotwords, metadata) to help transcription
|
||||
|
||||
Returns:
|
||||
BatchEncoding with:
|
||||
- input_ids: Token IDs for the model
|
||||
- attention_mask: Attention mask
|
||||
- acoustic_input_mask: Mask indicating speech token positions
|
||||
- speech_tensors: Processed speech features
|
||||
- speech_masks: Valid speech masks
|
||||
- vae_tok_seqlens: Length of each speech segment in tokens
|
||||
"""
|
||||
if audio is None:
|
||||
raise ValueError("Audio input is required for ASR processing")
|
||||
|
||||
# Handle single vs batch input
|
||||
if isinstance(audio, list):
|
||||
is_batched = True
|
||||
audio_list = audio
|
||||
else:
|
||||
is_batched = False
|
||||
audio_list = [audio]
|
||||
|
||||
# Process each audio input
|
||||
all_encodings = []
|
||||
for audio_input in audio_list:
|
||||
encoding = self._process_single_audio(
|
||||
audio_input,
|
||||
sampling_rate=sampling_rate,
|
||||
add_generation_prompt=add_generation_prompt,
|
||||
use_streaming=use_streaming,
|
||||
context_info=context_info,
|
||||
)
|
||||
all_encodings.append(encoding)
|
||||
|
||||
# Combine into batch
|
||||
batch_encoding = self._batch_encode(
|
||||
all_encodings,
|
||||
padding=padding,
|
||||
max_length=max_length,
|
||||
truncation=truncation,
|
||||
return_tensors=return_tensors,
|
||||
)
|
||||
|
||||
return batch_encoding
|
||||
|
||||
def _process_single_audio(
|
||||
self,
|
||||
audio: Union[str, np.ndarray, torch.Tensor],
|
||||
sampling_rate: Optional[int] = None,
|
||||
add_generation_prompt: bool = True,
|
||||
use_streaming: bool = True,
|
||||
context_info: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Process a single audio input.
|
||||
|
||||
Args:
|
||||
audio: Single audio input
|
||||
sampling_rate: Audio sampling rate
|
||||
add_generation_prompt: Whether to add generation prompt
|
||||
context_info: Optional context information (e.g., hotwords, metadata) to help transcription
|
||||
|
||||
Returns:
|
||||
Dictionary with processed tokens and audio features
|
||||
"""
|
||||
# Process audio through audio processor
|
||||
if isinstance(audio, str):
|
||||
# Load from file using ffmpeg for better format support
|
||||
if HAS_FFMPEG_UTILS:
|
||||
try:
|
||||
audio_array, file_sr = load_audio_use_ffmpeg(audio, resample=False)
|
||||
except Exception as e:
|
||||
# Fall back to soundfile if ffmpeg fails
|
||||
warnings.warn(f"ffmpeg loading failed, falling back to soundfile: {e}")
|
||||
import soundfile as sf
|
||||
audio_array, file_sr = sf.read(audio)
|
||||
if audio_array.ndim > 1:
|
||||
audio_array = audio_array.mean(axis=1) # Convert to mono
|
||||
else:
|
||||
import soundfile as sf
|
||||
audio_array, file_sr = sf.read(audio)
|
||||
if audio_array.ndim > 1:
|
||||
audio_array = audio_array.mean(axis=1) # Convert to mono
|
||||
|
||||
# Resample if needed
|
||||
if file_sr != self.target_sample_rate:
|
||||
import librosa
|
||||
audio_array = librosa.resample(
|
||||
audio_array,
|
||||
orig_sr=file_sr,
|
||||
target_sr=self.target_sample_rate
|
||||
)
|
||||
elif isinstance(audio, torch.Tensor):
|
||||
audio_array = audio.cpu().numpy()
|
||||
if audio_array.ndim > 1:
|
||||
audio_array = audio_array.squeeze()
|
||||
else:
|
||||
audio_array = np.array(audio, dtype=np.float32)
|
||||
if audio_array.ndim > 1:
|
||||
audio_array = audio_array.squeeze()
|
||||
|
||||
# Ensure float32
|
||||
audio_array = audio_array.astype(np.float32)
|
||||
|
||||
# Normalize if needed
|
||||
if self.normalize_audio and self.audio_normalizer:
|
||||
audio_array = self.audio_normalizer(audio_array)
|
||||
|
||||
# Calculate audio duration
|
||||
audio_duration = len(audio_array) / self.target_sample_rate
|
||||
|
||||
# Auto-disable streaming for short audio (<60s)
|
||||
if use_streaming and audio_duration < 60.0:
|
||||
use_streaming = False
|
||||
|
||||
# Calculate token length based on streaming mode
|
||||
# Non-streaming: uses ceil (encoder adds extra_padding for stride alignment)
|
||||
# Streaming: uses floor (segments processed independently, no global alignment)
|
||||
# if use_streaming:
|
||||
# vae_tok_len = len(audio_array) // self.speech_tok_compress_ratio
|
||||
# else:
|
||||
vae_tok_len = math.ceil(len(audio_array) / self.speech_tok_compress_ratio)
|
||||
|
||||
# Build token sequence following training format
|
||||
# 1. System prompt - use apply_chat_template then encode like in training
|
||||
system_prompt_text = self.tokenizer.apply_chat_template(
|
||||
[{"role": "system", "content": SYSTEM_PROMPT}],
|
||||
tokenize=False
|
||||
)
|
||||
system_tokens = self.tokenizer.encode(system_prompt_text)
|
||||
|
||||
# 2. User input with speech tokens
|
||||
# Build speech placeholder string
|
||||
sp_start_token = self.tokenizer.convert_ids_to_tokens(self.speech_start_id)
|
||||
sp_pad_token = self.tokenizer.convert_ids_to_tokens(self.speech_pad_id)
|
||||
sp_end_token = self.tokenizer.convert_ids_to_tokens(self.speech_end_id)
|
||||
|
||||
# User suffix with audio duration info
|
||||
show_keys = ['Start time', 'End time', 'Speaker ID', 'Content']
|
||||
if context_info and context_info.strip():
|
||||
user_suffix = f"This is a {audio_duration:.2f} seconds audio, with extra info: {context_info.strip()}\n\nPlease transcribe it with these keys: " + ", ".join(show_keys)
|
||||
else:
|
||||
user_suffix = f"This is a {audio_duration:.2f} seconds audio, please transcribe it with these keys: " + ", ".join(show_keys)
|
||||
|
||||
user_input_string = ''.join(
|
||||
[sp_start_token] + [sp_pad_token] * vae_tok_len + [sp_end_token]
|
||||
) + '\n' + user_suffix
|
||||
|
||||
user_tokens = self.tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": user_input_string}],
|
||||
tokenize=True
|
||||
)
|
||||
|
||||
# Combine tokens
|
||||
full_tokens = system_tokens + user_tokens
|
||||
|
||||
# Create acoustic input mask
|
||||
acoustic_input_mask = [1 if token == self.speech_pad_id else 0 for token in full_tokens]
|
||||
|
||||
return {
|
||||
"input_ids": full_tokens,
|
||||
"acoustic_input_mask": acoustic_input_mask,
|
||||
"speech": audio_array,
|
||||
"vae_tok_len": vae_tok_len,
|
||||
}
|
||||
|
||||
def _batch_encode(
|
||||
self,
|
||||
encodings: List[Dict[str, Any]],
|
||||
padding: bool = True,
|
||||
max_length: Optional[int] = None,
|
||||
truncation: bool = False,
|
||||
return_tensors: Optional[str] = None,
|
||||
) -> BatchEncoding:
|
||||
"""
|
||||
Combine multiple encodings into a batch.
|
||||
|
||||
Args:
|
||||
encodings: List of encoded samples
|
||||
padding: Whether to pad sequences
|
||||
max_length: Maximum sequence length
|
||||
truncation: Whether to truncate
|
||||
return_tensors: Output format
|
||||
|
||||
Returns:
|
||||
BatchEncoding with batched data
|
||||
"""
|
||||
# Extract components
|
||||
input_ids_list = [enc["input_ids"] for enc in encodings]
|
||||
acoustic_masks_list = [enc["acoustic_input_mask"] for enc in encodings]
|
||||
speech_list = [enc["speech"] for enc in encodings]
|
||||
vae_tok_lens = [enc["vae_tok_len"] for enc in encodings]
|
||||
|
||||
# Determine max length for padding
|
||||
if padding:
|
||||
if max_length is not None:
|
||||
target_length = max_length
|
||||
else:
|
||||
target_length = max(len(ids) for ids in input_ids_list)
|
||||
|
||||
# Pad sequences
|
||||
padded_input_ids = []
|
||||
padded_acoustic_masks = []
|
||||
attention_masks = []
|
||||
|
||||
for input_ids, acoustic_mask in zip(input_ids_list, acoustic_masks_list):
|
||||
# Truncate if needed
|
||||
if truncation and len(input_ids) > target_length:
|
||||
input_ids = input_ids[:target_length]
|
||||
acoustic_mask = acoustic_mask[:target_length]
|
||||
|
||||
# Pad sequences to left (for autoregressive generation)
|
||||
padding_length = target_length - len(input_ids)
|
||||
padded_ids = [self.pad_id] * padding_length + input_ids
|
||||
padded_acoustic = [0] * padding_length + acoustic_mask
|
||||
attention_mask = [0] * padding_length + [1] * len(input_ids)
|
||||
|
||||
padded_input_ids.append(padded_ids)
|
||||
padded_acoustic_masks.append(padded_acoustic)
|
||||
attention_masks.append(attention_mask)
|
||||
|
||||
input_ids_list = padded_input_ids
|
||||
acoustic_masks_list = padded_acoustic_masks
|
||||
else:
|
||||
attention_masks = [[1] * len(ids) for ids in input_ids_list]
|
||||
|
||||
# Process speech tensors - raw audio is 1D, so we keep it as is
|
||||
max_speech_length = max(len(s) for s in speech_list)
|
||||
padded_speeches = np.zeros((len(speech_list), max_speech_length), dtype=np.float32)
|
||||
speech_masks = np.zeros((len(speech_list), max(vae_tok_lens)), dtype=bool)
|
||||
|
||||
for i, (speech, vae_len) in enumerate(zip(speech_list, vae_tok_lens)):
|
||||
padded_speeches[i, :len(speech)] = speech
|
||||
speech_masks[i, :vae_len] = True
|
||||
|
||||
# Create batch encoding
|
||||
batch_encoding = BatchEncoding()
|
||||
|
||||
if return_tensors == "pt":
|
||||
batch_encoding["input_ids"] = torch.tensor(input_ids_list, dtype=torch.long)
|
||||
batch_encoding["attention_mask"] = torch.tensor(attention_masks, dtype=torch.long)
|
||||
batch_encoding["acoustic_input_mask"] = torch.tensor(acoustic_masks_list, dtype=torch.bool)
|
||||
batch_encoding["speech_tensors"] = torch.tensor(padded_speeches, dtype=torch.float32)
|
||||
batch_encoding["speech_masks"] = torch.tensor(speech_masks, dtype=torch.bool)
|
||||
# Note: vae_tok_seqlens and speech_type are not included as they are not model inputs
|
||||
else:
|
||||
batch_encoding["input_ids"] = input_ids_list if len(input_ids_list) > 1 else input_ids_list[0]
|
||||
batch_encoding["attention_mask"] = attention_masks if len(attention_masks) > 1 else attention_masks[0]
|
||||
batch_encoding["acoustic_input_mask"] = acoustic_masks_list if len(acoustic_masks_list) > 1 else acoustic_masks_list[0]
|
||||
batch_encoding["speech_tensors"] = padded_speeches if len(padded_speeches) > 1 else padded_speeches[0]
|
||||
batch_encoding["speech_masks"] = speech_masks if len(speech_masks) > 1 else speech_masks[0]
|
||||
|
||||
return batch_encoding
|
||||
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""
|
||||
Decode batch of token IDs to text.
|
||||
Forwards to tokenizer's batch_decode method.
|
||||
"""
|
||||
return self.tokenizer.batch_decode(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
"""
|
||||
Decode token IDs to text.
|
||||
Forwards to tokenizer's decode method.
|
||||
"""
|
||||
return self.tokenizer.decode(*args, **kwargs)
|
||||
|
||||
def post_process_transcription(self, text: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Post-process the generated transcription text to extract structured data.
|
||||
|
||||
Args:
|
||||
text: Generated text from the model
|
||||
|
||||
Returns:
|
||||
List of dictionaries with transcription segments
|
||||
"""
|
||||
try:
|
||||
# Try to parse as JSON
|
||||
if "```json" in text:
|
||||
# Extract JSON from markdown code block
|
||||
json_start = text.find("```json") + 7
|
||||
json_end = text.find("```", json_start)
|
||||
json_str = text[json_start:json_end].strip()
|
||||
else:
|
||||
# Try to find JSON array or object
|
||||
json_start = text.find("[")
|
||||
if json_start == -1:
|
||||
json_start = text.find("{")
|
||||
if json_start != -1:
|
||||
# Find matching closing bracket
|
||||
bracket_count = 0
|
||||
json_end = json_start
|
||||
for i in range(json_start, len(text)):
|
||||
if text[i] in "[{":
|
||||
bracket_count += 1
|
||||
elif text[i] in "]}":
|
||||
bracket_count -= 1
|
||||
if bracket_count == 0:
|
||||
json_end = i + 1
|
||||
break
|
||||
json_str = text[json_start:json_end]
|
||||
else:
|
||||
json_str = text
|
||||
|
||||
# Parse JSON
|
||||
result = json.loads(json_str)
|
||||
|
||||
# Ensure it's a list
|
||||
if isinstance(result, dict):
|
||||
result = [result]
|
||||
|
||||
# Validate and clean up the result
|
||||
cleaned_result = []
|
||||
for item in result:
|
||||
if isinstance(item, dict):
|
||||
cleaned_item = {}
|
||||
# Map keys to expected format
|
||||
key_mapping = {
|
||||
"Start time": "start_time",
|
||||
"Start": "start_time",
|
||||
"End time": "end_time",
|
||||
"End": "end_time",
|
||||
"Speaker ID": "speaker_id",
|
||||
"Speaker": "speaker_id",
|
||||
"Content": "text",
|
||||
}
|
||||
for key, mapped_key in key_mapping.items():
|
||||
if key in item:
|
||||
cleaned_item[mapped_key] = item[key]
|
||||
|
||||
if cleaned_item:
|
||||
cleaned_result.append(cleaned_item)
|
||||
|
||||
return cleaned_result
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(f"Failed to parse JSON from transcription: {e}")
|
||||
logger.debug(f"Raw text: {text}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.warning(f"Error post-processing transcription: {e}")
|
||||
return []
|
||||
|
||||
@property
|
||||
def model_input_names(self):
|
||||
"""Return the list of inputs accepted by the model."""
|
||||
return ["input_ids", "attention_mask", "acoustic_input_mask", "speech_tensors", "speech_masks"]
|
||||
|
||||
__all__ = ["VibeVoiceASRProcessor"]
|
||||
Reference in New Issue
Block a user