|
| 1 | +import logging |
| 2 | +import os |
| 3 | +import uuid |
| 4 | + |
| 5 | +import soundfile as sf |
| 6 | +import torch |
| 7 | +from datasets import load_from_disk |
| 8 | +from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor |
| 9 | + |
| 10 | +from ts.torch_handler.base_handler import BaseHandler |
| 11 | + |
| 12 | +logger = logging.getLogger(__name__) |
| 13 | + |
| 14 | + |
| 15 | +class SpeechT5_TTS(BaseHandler): |
| 16 | + def __init__(self): |
| 17 | + self.model = None |
| 18 | + self.processor = None |
| 19 | + self.vocoder = None |
| 20 | + self.speaker_embeddings = None |
| 21 | + self.output_dir = "/tmp" |
| 22 | + |
| 23 | + def initialize(self, ctx): |
| 24 | + properties = ctx.system_properties |
| 25 | + model_dir = properties.get("model_dir") |
| 26 | + |
| 27 | + processor = ctx.model_yaml_config["handler"]["processor"] |
| 28 | + model = ctx.model_yaml_config["handler"]["model"] |
| 29 | + vocoder = ctx.model_yaml_config["handler"]["vocoder"] |
| 30 | + embeddings_dataset = ctx.model_yaml_config["handler"]["speaker_embeddings"] |
| 31 | + self.output_dir = ctx.model_yaml_config["handler"]["output_dir"] |
| 32 | + |
| 33 | + self.processor = SpeechT5Processor.from_pretrained(processor) |
| 34 | + self.model = SpeechT5ForTextToSpeech.from_pretrained(model) |
| 35 | + self.vocoder = SpeechT5HifiGan.from_pretrained(vocoder) |
| 36 | + |
| 37 | + # load xvector containing speaker's voice characteristics from a dataset |
| 38 | + embeddings_dataset = load_from_disk(embeddings_dataset) |
| 39 | + self.speaker_embeddings = torch.tensor( |
| 40 | + embeddings_dataset[7306]["xvector"] |
| 41 | + ).unsqueeze(0) |
| 42 | + |
| 43 | + def preprocess(self, requests): |
| 44 | + assert len(requests) == 1, "This is currently supported with batch_size=1" |
| 45 | + req_data = requests[0] |
| 46 | + |
| 47 | + input_data = req_data.get("data") or req_data.get("body") |
| 48 | + |
| 49 | + if isinstance(input_data, (bytes, bytearray)): |
| 50 | + input_data = input_data.decode("utf-8") |
| 51 | + |
| 52 | + inputs = self.processor(text=input_data, return_tensors="pt") |
| 53 | + |
| 54 | + return inputs |
| 55 | + |
| 56 | + def inference(self, inputs): |
| 57 | + output = self.model.generate_speech( |
| 58 | + inputs["input_ids"], self.speaker_embeddings, vocoder=self.vocoder |
| 59 | + ) |
| 60 | + return output |
| 61 | + |
| 62 | + def postprocess(self, inference_output): |
| 63 | + path = self.output_dir + "/{}.wav".format(uuid.uuid4().hex) |
| 64 | + sf.write(path, inference_output.numpy(), samplerate=16000) |
| 65 | + with open(path, "rb") as output: |
| 66 | + data = output.read() |
| 67 | + os.remove(path) |
| 68 | + return [data] |
0 commit comments