Merge pull request w-okada#371 from nadare881/RVC_improve

リアルタイム特化推論の追加

server/voice_changer/RVC/RVC.py

@@ -23,7 +23,7 @@
 
 from voice_changer.RVC.RVCSettings import RVCSettings
 from voice_changer.RVC.embedder.EmbedderManager import EmbedderManager
-from voice_changer.utils.VoiceChangerModel import AudioInOut, VoiceChangerModel
+from voice_changer.utils.VoiceChangerModel import AudioInOut, PitchfInOut, FeatureInOut, VoiceChangerModel
 from voice_changer.utils.VoiceChangerParams import VoiceChangerParams
 from voice_changer.RVC.onnxExporter.export2onnx import export2onnx
 from voice_changer.RVC.pitchExtractor.PitchExtractorManager import PitchExtractorManager
@@ -46,6 +46,8 @@ def __init__(self, params: VoiceChangerParams, slotInfo: RVCModelSlot):
         self.pipeline: Pipeline | None = None
 
         self.audio_buffer: AudioInOut | None = None
+        self.pitchf_buffer: PitchfInOut | None = None
+        self.feature_buffer: FeatureInOut | None = None
         self.prevVol = 0.0
         self.slotInfo = slotInfo
         self.initialize()
@@ -99,49 +101,65 @@ def generate_input(
     ):
         newData = newData.astype(np.float32) / 32768.0  # RVCのモデルのサンプリングレートで入ってきている。（extraDataLength, Crossfade等も同じSRで処理）(★１)
 
+        new_feature_length = newData.shape[0] * 100 // self.slotInfo.samplingRate
         if self.audio_buffer is not None:
             # 過去のデータに連結
             self.audio_buffer = np.concatenate([self.audio_buffer, newData], 0)
+            if self.slotInfo.f0:
+                self.pitchf_buffer = np.concatenate([self.pitchf_buffer, np.zeros(new_feature_length)], 0)
+            self.feature_buffer = np.concatenate([self.feature_buffer, np.zeros([new_feature_length, self.slotInfo.embChannels])], 0)
         else:
             self.audio_buffer = newData
+            if self.slotInfo.f0:
+                self.pitchf_buffer = np.zeros(new_feature_length)
+            self.feature_buffer =  np.zeros([new_feature_length, self.slotInfo.embChannels])
 
         convertSize = inputSize + crossfadeSize + solaSearchFrame + self.settings.extraConvertSize
 
         if convertSize % 128 != 0:  # モデルの出力のホップサイズで切り捨てが発生するので補う。
             convertSize = convertSize + (128 - (convertSize % 128))
+        outSize = convertSize - self.settings.extraConvertSize
 
         # バッファがたまっていない場合はzeroで補う
         if self.audio_buffer.shape[0] < convertSize:
             self.audio_buffer = np.concatenate([np.zeros([convertSize]), self.audio_buffer])
+            if self.slotInfo.f0:
+                self.pitchf_buffer = np.concatenate([np.zeros([convertSize * 100 // self.slotInfo.samplingRate]), self.pitchf_buffer])
+            self.feature_buffer = np.concatenate([np.zeros([convertSize * 100 // self.slotInfo.samplingRate, self.slotInfo.embChannels]), self.feature_buffer])
 
         convertOffset = -1 * convertSize
+        featureOffset = -convertSize * 100 // self.slotInfo.samplingRate
         self.audio_buffer = self.audio_buffer[convertOffset:]  # 変換対象の部分だけ抽出
-
-        if self.pipeline is not None:
-            device = self.pipeline.device
-        else:
-            device = torch.device("cpu")
-
-        audio_buffer = torch.from_numpy(self.audio_buffer).to(device=device, dtype=torch.float32)
-
+        if self.slotInfo.f0:
+            self.pitchf_buffer = self.pitchf_buffer[featureOffset:]
+        self.feature_buffer = self.feature_buffer[featureOffset:]
+
         # 出力部分だけ切り出して音量を確認。(TODO:段階的消音にする)
         cropOffset = -1 * (inputSize + crossfadeSize)
         cropEnd = -1 * (crossfadeSize)
-        crop = audio_buffer[cropOffset:cropEnd]
-        vol = torch.sqrt(torch.square(crop).mean()).detach().cpu().numpy()
+        crop = self.audio_buffer[cropOffset:cropEnd]
+        vol = np.sqrt(np.square(crop).mean())
         vol = max(vol, self.prevVol * 0.0)
         self.prevVol = vol
 
-        return (audio_buffer, convertSize, vol)
+        return (self.audio_buffer, self.pitchf_buffer, self.feature_buffer, convertSize, vol, outSize)
 
     def inference(self, data):
         audio = data[0]
-        convertSize = data[1]
-        vol = data[2]
+        pitchf = data[1]
+        feature = data[2]
+        convertSize = data[3]
+        vol = data[4]
+        outSize = data[5]
 
         if vol < self.settings.silentThreshold:
-            return np.zeros(convertSize).astype(np.int16)
+            return np.zeros(convertSize).astype(np.int16) * np.sqrt(vol)
 
+        if self.pipeline is not None:
+            device = self.pipeline.device
+        else:
+            device = torch.device("cpu")
+        audio = torch.from_numpy(audio).to(device=device, dtype=torch.float32)
         audio = torchaudio.functional.resample(audio, self.slotInfo.samplingRate, 16000, rolloff=0.99)
         repeat = 1 if self.settings.rvcQuality else 0
         sid = 0
@@ -154,17 +172,20 @@ def inference(self, data):
         useFinalProj = self.slotInfo.useFinalProj
 
         try:
-            audio_out = self.pipeline.exec(
+            audio_out, self.pitchf_buffer, self.feature_buffer = self.pipeline.exec(
                 sid,
                 audio,
+                pitchf,
+                feature,
                 f0_up_key,
                 index_rate,
                 if_f0,
-                self.settings.extraConvertSize / self.slotInfo.samplingRate,  # extaraDataSizeの秒数。RVCのモデルのサンプリングレートで処理(★１)。
+                self.settings.extraConvertSize / self.slotInfo.samplingRate if self.settings.silenceFront else 0.,  # extaraDataSizeの秒数。RVCのモデルのサンプリングレートで処理(★１)。
                 embOutputLayer,
                 useFinalProj,
                 repeat,
                 protect,
+                outSize
             )
             result = audio_out.detach().cpu().numpy() * np.sqrt(vol)
 

server/voice_changer/RVC/inferencer/OnnxRVCInferencer.py

@@ -35,6 +35,7 @@ def infer(
         pitch: torch.Tensor,
         pitchf: torch.Tensor,
         sid: torch.Tensor,
+        convert_length: int | None,
     ) -> torch.Tensor:
         if pitch is None or pitchf is None:
             raise RuntimeError("[Voice Changer] Pitch or Pitchf is not found.")
@@ -50,7 +51,7 @@ def infer(
                     "p_len": pitch_length.cpu().numpy().astype(np.int64),
                     "pitch": pitch.cpu().numpy().astype(np.int64),
                     "pitchf": pitchf.cpu().numpy().astype(np.float32),
-                    "sid": sid.cpu().numpy().astype(np.int64),
+                    "sid": sid.cpu().numpy().astype(np.int64)               
                 },
             )
         else:
@@ -61,7 +62,7 @@ def infer(
                     "p_len": pitch_length.cpu().numpy().astype(np.int64),
                     "pitch": pitch.cpu().numpy().astype(np.int64),
                     "pitchf": pitchf.cpu().numpy().astype(np.float32),
-                    "sid": sid.cpu().numpy().astype(np.int64),
+                    "sid": sid.cpu().numpy().astype(np.int64) 
                 },
             )
 

server/voice_changer/RVC/inferencer/OnnxRVCInferencerNono.py

@@ -4,7 +4,6 @@
 
 from voice_changer.RVC.inferencer.OnnxRVCInferencer import OnnxRVCInferencer
 
-
 class OnnxRVCInferencerNono(OnnxRVCInferencer):
     def loadModel(self, file: str, gpu: int):
         super().loadModel(file, gpu)
@@ -18,6 +17,7 @@ def infer(
         pitch: torch.Tensor | None,
         pitchf: torch.Tensor | None,
         sid: torch.Tensor,
+        convert_length: int | None,
     ) -> torch.Tensor:
         if self.isHalf:
             audio1 = self.model.run(

server/voice_changer/RVC/inferencer/RVCInferencer.py

@@ -33,5 +33,6 @@ def infer(
         pitch: torch.Tensor,
         pitchf: torch.Tensor,
         sid: torch.Tensor,
+        convert_length: int | None,
     ) -> torch.Tensor:
-        return self.model.infer(feats, pitch_length, pitch, pitchf, sid)
+        return self.model.infer(feats, pitch_length, pitch, pitchf, sid, convert_length=convert_length)

server/voice_changer/RVC/inferencer/RVCInferencerNono.py

@@ -33,5 +33,6 @@ def infer(
         pitch: torch.Tensor | None,
         pitchf: torch.Tensor | None,
         sid: torch.Tensor,
+        convert_length: int | None,
     ) -> torch.Tensor:
-        return self.model.infer(feats, pitch_length, sid)
+        return self.model.infer(feats, pitch_length, sid, convert_length=convert_length)

server/voice_changer/RVC/inferencer/RVCInferencerv2.py

@@ -32,5 +32,6 @@ def infer(
         pitch: torch.Tensor,
         pitchf: torch.Tensor,
         sid: torch.Tensor,
+        convert_length: int | None,
     ) -> torch.Tensor:
-        return self.model.infer(feats, pitch_length, pitch, pitchf, sid)
+        return self.model.infer(feats, pitch_length, pitch, pitchf, sid, convert_length=convert_length)

server/voice_changer/RVC/inferencer/RVCInferencerv2Nono.py

@@ -33,5 +33,6 @@ def infer(
         pitch: torch.Tensor | None,
         pitchf: torch.Tensor | None,
         sid: torch.Tensor,
+        convert_length: int | None,
     ) -> torch.Tensor:
-        return self.model.infer(feats, pitch_length, sid)
+        return self.model.infer(feats, pitch_length, sid, convert_length=convert_length)

server/voice_changer/RVC/inferencer/VorasInferencebeta.py

@@ -35,5 +35,6 @@ def infer(
         pitch: torch.Tensor,
         pitchf: torch.Tensor,
         sid: torch.Tensor,
+        convert_length: int | None,
     ) -> torch.Tensor:
         return self.model.infer(feats, pitch_length, pitch, pitchf, sid)

server/voice_changer/RVC/inferencer/WebUIInferencer.py

@@ -33,5 +33,6 @@ def infer(
         pitch: torch.Tensor,
         pitchf: torch.Tensor,
         sid: torch.Tensor,
+        convert_length: int | None,
     ) -> torch.Tensor:
-        return self.model.infer(feats, pitch_length, pitch, pitchf, sid)
+        return self.model.infer(feats, pitch_length, pitch, pitchf, sid, convert_length=convert_length)

server/voice_changer/RVC/inferencer/WebUIInferencerNono.py

@@ -33,5 +33,6 @@ def infer(
         pitch: torch.Tensor | None,
         pitchf: torch.Tensor | None,
         sid: torch.Tensor,
+        convert_length: int | None,
     ) -> torch.Tensor:
-        return self.model.infer(feats, pitch_length, sid)
+        return self.model.infer(feats, pitch_length, sid, convert_length=convert_length)

server/voice_changer/RVC/inferencer/models.py

@@ -129,15 +129,16 @@ def forward(self, phone, phone_lengths, pitch, pitchf, y, y_lengths, ds):  # 这
         o = self.dec(z_slice, pitchf, g=g)
         return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
 
-    def infer(self, phone, phone_lengths, pitch, nsff0, sid, max_len=None):
+    def infer(self, phone, phone_lengths, pitch, nsff0, sid, max_len=None, convert_length=None):
         g = self.emb_g(sid).unsqueeze(-1)
         m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)
         z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
         z = self.flow(z_p, x_mask, g=g, reverse=True)
-        o = self.dec((z * x_mask)[:, :, :max_len], nsff0, g=g)
+        o = self.dec.infer_realtime((z * x_mask)[:, :, :max_len], nsff0, g=g, convert_length=convert_length)
         return o, x_mask, (z, z_p, m_p, logs_p)
 
 
+
 class SynthesizerTrnMsNSFsidNono(nn.Module):
     def __init__(self, spec_channels, segment_size, inter_channels, hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, spk_embed_dim, gin_channels, emb_channels, sr=None, **kwargs):
         super().__init__()
@@ -208,10 +209,10 @@ def forward(self, phone, phone_lengths, y, y_lengths, ds):  # 这里ds是id，[b
         o = self.dec(z_slice, g=g)
         return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
 
-    def infer(self, phone, phone_lengths, sid, max_len=None):
+    def infer(self, phone, phone_lengths, sid, max_len=None, convert_length=None):
         g = self.emb_g(sid).unsqueeze(-1)
         m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths)
         z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
         z = self.flow(z_p, x_mask, g=g, reverse=True)
-        o = self.dec((z * x_mask)[:, :, :max_len], g=g)
+        o = self.dec.infer_realtime((z * x_mask)[:, :, :max_len], g=g, convert_length=convert_length)
         return o, x_mask, (z, z_p, m_p, logs_p)