Gabriel de Marmiesse
c90fc8ced6
The french 6 layers variant wasn't included because it's not good enough yet. Use the `french_24l` variant for now.
120 lines
4.3 KiB
Python
120 lines
4.3 KiB
Python
import logging
|
|
|
|
import torch
|
|
from torch import nn
|
|
|
|
from pocket_tts.modules.conv import pad_for_conv1d
|
|
from pocket_tts.modules.dummy_quantizer import DummyQuantizer
|
|
from pocket_tts.modules.mimi_transformer import ProjectedTransformer
|
|
from pocket_tts.modules.resample import ConvDownsample1d, ConvTrUpsample1d
|
|
from pocket_tts.modules.seanet import SEANetDecoder, SEANetEncoder
|
|
|
|
logger = logging.getLogger()
|
|
|
|
|
|
class MimiModel(nn.Module):
|
|
def __init__(
|
|
self,
|
|
encoder: SEANetEncoder,
|
|
decoder: SEANetDecoder,
|
|
quantizer: DummyQuantizer,
|
|
frame_rate: float,
|
|
encoder_frame_rate: float,
|
|
sample_rate: int,
|
|
channels: int,
|
|
inner_dim: int | None,
|
|
outer_dim: int | None,
|
|
encoder_transformer: ProjectedTransformer,
|
|
decoder_transformer: ProjectedTransformer,
|
|
):
|
|
super().__init__()
|
|
self.encoder = encoder
|
|
self.decoder = decoder
|
|
self.encoder_transformer = encoder_transformer
|
|
self.decoder_transformer = decoder_transformer
|
|
self.quantizer = quantizer
|
|
self.frame_rate = frame_rate
|
|
self.sample_rate = sample_rate
|
|
self.channels = channels
|
|
self.encoder_frame_rate = encoder_frame_rate
|
|
|
|
# We will need the dimension for the resampling. In general the encoder will be a SeanetEncoder
|
|
# which exposes a `dimension` attribute.
|
|
dimension = encoder.dimension
|
|
assert isinstance(dimension, int), (
|
|
f"Dimension should be int, got {dimension} of type {type(dimension)}."
|
|
)
|
|
self.dimension = dimension
|
|
|
|
if encoder_frame_rate != frame_rate:
|
|
assert self.encoder_frame_rate > self.frame_rate, "Cannot upsample with conv."
|
|
downsample_stride = self.encoder_frame_rate / self.frame_rate
|
|
assert downsample_stride == int(downsample_stride), (
|
|
f"Only integer strides are supported, got {downsample_stride}"
|
|
)
|
|
|
|
# v2 of models
|
|
self.downsample = ConvDownsample1d(
|
|
int(downsample_stride), dimension=dimension, out_dimension=inner_dim
|
|
)
|
|
self.upsample = ConvTrUpsample1d(
|
|
int(downsample_stride), dimension=dimension, in_dimension=outer_dim
|
|
)
|
|
|
|
@property
|
|
def frame_size(self) -> int:
|
|
return int(self.sample_rate / self.frame_rate)
|
|
|
|
def _to_framerate(self, x: torch.Tensor):
|
|
# Convert from the encoder frame rate to the overall framerate.
|
|
_, _, length = x.shape
|
|
frame_rate = self.encoder_frame_rate
|
|
new_frame_rate = self.frame_rate
|
|
if frame_rate == new_frame_rate:
|
|
return x
|
|
return self.downsample(x, model_state=None)
|
|
|
|
def _to_encoder_framerate(self, x: torch.Tensor, mimi_state) -> torch.Tensor:
|
|
# Convert from overall framerate to the encoder frame rate.
|
|
_, _, length = x.shape
|
|
frame_rate = self.encoder_frame_rate
|
|
new_frame_rate = self.frame_rate
|
|
if frame_rate == new_frame_rate:
|
|
return x
|
|
return self.upsample(x, mimi_state)
|
|
|
|
def forward(self, x: torch.Tensor):
|
|
raise NotImplementedError()
|
|
|
|
def decode_from_latent(self, latent: torch.Tensor, mimi_state) -> torch.Tensor:
|
|
emb = self._to_encoder_framerate(latent, mimi_state)
|
|
(emb,) = self.decoder_transformer(emb, mimi_state)
|
|
out = self.decoder(emb, mimi_state)
|
|
# out contains extra padding added by the encoder and decoder
|
|
return out
|
|
|
|
def encode_to_latent(self, x: torch.Tensor) -> torch.Tensor:
|
|
"""Projects a batch of waveforms to unquantized latent space.
|
|
|
|
Args:
|
|
x (torch.Tensor): Float tensor of shape [B, C, T].
|
|
|
|
Returns:
|
|
Unquantized embeddings.
|
|
"""
|
|
assert x.dim() == 3, (
|
|
f"CompressionModel._encode_to_unquantized_latent expects audio of shape [B, C, T] but got {x.shape}"
|
|
)
|
|
|
|
frame_size = self.frame_size
|
|
|
|
# The underlying convolutions no longer accept partial inputs,
|
|
# `x` needs to be exactly a multiple of the frame size,
|
|
# reproducing the previous padding behavior here.
|
|
x = pad_for_conv1d(x, frame_size, frame_size)
|
|
emb = self.encoder(x, model_state=None)
|
|
|
|
(emb,) = self.encoder_transformer(emb, model_state=None)
|
|
emb = self._to_framerate(emb)
|
|
return emb
|