Spaces:
Sleeping
Sleeping
File size: 3,001 Bytes
df2accb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
# Copyright (c) 2023 Amphion.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import torch
class PromptPreparer:
def prepare_prompts(self, y, y_lens, codes, nar_stage, y_prompts_codes):
if self.prefix_mode == 0:
y_emb, prefix_len = self._handle_prefix_mode_0(y, codes, nar_stage)
elif self.prefix_mode == 1:
y_emb, prefix_len = self._handle_prefix_mode_1(y, y_lens, codes, nar_stage)
elif self.prefix_mode in [2, 4]:
y_emb, prefix_len = self._handle_prefix_mode_2_4(y, y_lens, codes, nar_stage, y_prompts_codes)
else:
raise ValueError("Invalid prefix mode")
return y_emb, prefix_len
def _handle_prefix_mode_0(self, y, codes, nar_stage):
prefix_len = 0
y_emb = self.nar_audio_embeddings[0](y)
for j in range(1, nar_stage):
y_emb = y_emb + self.nar_audio_embeddings[j](codes[..., j])
return y_emb, 0
def _handle_prefix_mode_1(self, y, y_lens, codes, nar_stage):
int_low = (0.25 * y_lens.min()).type(torch.int64).item()
prefix_len = torch.randint(int_low, int_low * 2, size=()).item()
prefix_len = min(prefix_len, 225)
y_prompts = self.nar_audio_embeddings[0](y[:, :prefix_len])
y_emb = self.nar_audio_embeddings[0](y[:, prefix_len:])
for j in range(1, self.num_quantizers):
y_prompts += self.nar_audio_embeddings[j](
codes[:, :prefix_len, j]
)
if j < nar_stage:
y_emb += self.nar_audio_embeddings[j](
codes[:, prefix_len:, j]
)
y_emb = torch.concat([y_prompts, y_emb], axis=1)
return y_emb, prefix_len
def _handle_prefix_mode_2_4(self, y, y_lens, codes, nar_stage, y_prompts_codes):
if self.prefix_mode == 2:
prefix_len = min(225, int(0.25 * y_lens.min().item()))
y_prompts_codes = []
for b in range(codes.shape[0]):
start = self.rng.randint(0, y_lens[b].item() - prefix_len)
y_prompts_codes.append(
torch.clone(codes[b, start : start + prefix_len])
)
codes[
b, start : start + prefix_len, nar_stage
] = self.audio_token_num
y_prompts_codes = torch.stack(y_prompts_codes, dim=0)
else:
prefix_len = y_prompts_codes.shape[1]
y_prompts = self.nar_audio_embeddings[0](y_prompts_codes[..., 0])
y_emb = self.nar_audio_embeddings[0](y)
for j in range(1, self.num_quantizers):
y_prompts += self.nar_audio_embeddings[j](
y_prompts_codes[..., j]
)
if j < nar_stage:
y_emb += self.nar_audio_embeddings[j](codes[..., j])
y_emb = torch.concat([y_prompts, y_emb], axis=1)
return y_emb, prefix_len
|