From 2dcf51668c9a290ebd4e5f640cd657c074d3195f Mon Sep 17 00:00:00 2001 From: cadlaxa Date: Mon, 9 Sep 2024 11:40:29 +0800 Subject: [PATCH 1/2] [DIFFS EN+] fixes to phoneme replacements with LangCode --- .../DiffSingerARPAPlusEnglishPhonemizer.cs | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerARPAPlusEnglishPhonemizer.cs b/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerARPAPlusEnglishPhonemizer.cs index 7332b3bc9..f350c8b84 100644 --- a/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerARPAPlusEnglishPhonemizer.cs +++ b/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerARPAPlusEnglishPhonemizer.cs @@ -5,6 +5,8 @@ using System.Text; using OpenUtau.Api; using OpenUtau.Core.G2p; +using OpenUtau.Core.Ustx; +using Serilog; namespace OpenUtau.Core.DiffSinger { [Phonemizer("DiffSinger English+ Phonemizer", "DIFFS EN+", language: "EN", author: "Cadlaxa")] @@ -37,15 +39,16 @@ public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevN throw new Exception("Result not found in the part"); } var processedPhonemes = new List(); + var langCode = GetLangCode() + "/"; for (int i = 0; i < phonemes.Count; i++) { var tu = phonemes[i]; // Check for "n dx" sequence and replace it with "n" // the actual phoneme for this is "nx" like (winner [w ih nx er]) - if (i < phonemes.Count - 1 && tu.Item1 == "n" && phonemes[i + 1].Item1 == "dx") { + if (i < phonemes.Count - 1 && tu.Item1 == langCode + "n" && phonemes[i + 1].Item1 == langCode + "dx") { processedPhonemes.Add(new Phoneme() { - phoneme = "n", + phoneme = langCode + "n", position = tu.Item2 }); // Skip the next phoneme ("dx") @@ -70,28 +73,48 @@ public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevN // Method to determine if a phoneme should be replaced based on specific conditions private bool ShouldReplacePhoneme(string phoneme, Note? prev, Note? next, Note? prevNeighbour, Note? nextNeighbour, out string replacement) { replacement = phoneme; - if (phoneme == "q") { - replacement = "cl"; - return true; - } - if (phoneme == "q") { - // vocal fry the vowel is the prevNeighbour is null + var langCode = GetLangCode() + "/"; + + if (phoneme == langCode + "q" || phoneme == "cl") { + // Vocal fry the vowel if the prevNeighbour is null if (!prevNeighbour.HasValue || string.IsNullOrWhiteSpace(prevNeighbour.Value.lyric)) { - replacement = "vf"; - return true; + replacement = "vf"; + return true; } } - // automatic relaxed consonants - if ((phoneme == "t" || phoneme == "d") && (nextNeighbour.HasValue && IsVowel(nextNeighbour.Value))) { - replacement = "dx"; + + if (phoneme == langCode + "q") { + // Replace "q" with "cl" + replacement = "cl"; + return true; + } + + // (doesn't work currently) Automatic relaxed consonants: replace "t" or "d" with "dx" if nextNeighbour is a vowel + if ((phoneme == langCode + "t" || phoneme == langCode + "d") && nextNeighbour.HasValue && IsVowel(nextNeighbour.Value)) { + replacement = langCode + "dx"; return true; } return false; } + // Method to check if a phoneme is a vowel private bool IsVowel(Note note) { - string[] vowels = GetBaseG2pVowels(); - return vowels.Contains(note.lyric); + // Retrieve the language code and vowel list + var langCode = GetLangCode(); + string[] baseVowels = GetBaseG2pVowels(); + + // Create the full list of vowel phonemes with language code + string[] vowels = baseVowels.Select(vowel => $"{langCode}/{vowel}").ToArray(); + + // Combine langCode and note.lyric to get the phoneme to check + string phonemeToCheck = $"{langCode}/{note.lyric}"; + + // Log the phoneme being checked and the list of vowels + Log.Debug($"IsVowel: Checking phoneme={phonemeToCheck}"); + Log.Debug($"IsVowel: Available vowels={string.Join(", ", vowels)}"); + + // Check if the phonemeToCheck is in the list of vowels + return vowels.Contains(phonemeToCheck); } } } From 88b550f30d7915cd471745df3265b434cfd0905a Mon Sep 17 00:00:00 2001 From: cadlaxa Date: Mon, 11 Nov 2024 21:58:00 +0800 Subject: [PATCH 2/2] [DIFFS EN+] condition fixes and add `HasPhoneme` function on `DiffSingerG2pPhonemizer` --- .../DiffSingerARPAPlusEnglishPhonemizer.cs | 67 ++++++++++--------- .../Phonemizers/DiffSingerG2pPhonemizer.cs | 18 ++++- 2 files changed, 50 insertions(+), 35 deletions(-) diff --git a/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerARPAPlusEnglishPhonemizer.cs b/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerARPAPlusEnglishPhonemizer.cs index f350c8b84..5da60666a 100644 --- a/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerARPAPlusEnglishPhonemizer.cs +++ b/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerARPAPlusEnglishPhonemizer.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Reflection; using System.Text; using OpenUtau.Api; using OpenUtau.Core.G2p; @@ -46,19 +47,28 @@ public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevN // Check for "n dx" sequence and replace it with "n" // the actual phoneme for this is "nx" like (winner [w ih nx er]) - if (i < phonemes.Count - 1 && tu.Item1 == langCode + "n" && phonemes[i + 1].Item1 == langCode + "dx") { + if (i < phonemes.Count - 1 && tu.Item1 == langCode + "n" && HasPhoneme(langCode + "n") && phonemes[i + 1].Item1 == langCode + "dx" && HasPhoneme(langCode + "dx")) { + // If phoneme "n" and "dx" exist, process "n" and skip "dx" processedPhonemes.Add(new Phoneme() { phoneme = langCode + "n", position = tu.Item2 }); - // Skip the next phoneme ("dx") + i++; // Skip next phoneme + } else if (i < phonemes.Count - 1 && tu.Item1 == "n" && HasPhoneme("n") && phonemes[i + 1].Item1 == "dx" && HasPhoneme("dx") && !HasPhoneme(langCode + "n") && !HasPhoneme(langCode + "dx")) { + // If phoneme "n" and "dx" exist, but language-specific "n" and "dx" don't exist, process "n" + processedPhonemes.Add(new Phoneme() { + phoneme = "n", + position = tu.Item2 + }); i++; } else if (ShouldReplacePhoneme(tu.Item1, prev, next, prevNeighbour, nextNeighbour, out string replacement)) { + // If phoneme should be replaced, process the replacement processedPhonemes.Add(new Phoneme() { phoneme = replacement, position = tu.Item2 }); } else { + // If no conditions are met, just add the current phoneme processedPhonemes.Add(new Phoneme() { phoneme = tu.Item1, position = tu.Item2 @@ -70,51 +80,44 @@ public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevN }; } - // Method to determine if a phoneme should be replaced based on specific conditions private bool ShouldReplacePhoneme(string phoneme, Note? prev, Note? next, Note? prevNeighbour, Note? nextNeighbour, out string replacement) { replacement = phoneme; var langCode = GetLangCode() + "/"; - if (phoneme == langCode + "q" || phoneme == "cl") { - // Vocal fry the vowel if the prevNeighbour is null + if ((phoneme == langCode + "cl" || !HasPhoneme("q")) && HasPhoneme("vf")) { if (!prevNeighbour.HasValue || string.IsNullOrWhiteSpace(prevNeighbour.Value.lyric)) { replacement = "vf"; return true; } } - - if (phoneme == langCode + "q") { - // Replace "q" with "cl" + if ((phoneme == langCode + "q" || phoneme == "q") && HasPhoneme("vf")) { + if (!prevNeighbour.HasValue || string.IsNullOrWhiteSpace(prevNeighbour.Value.lyric)) { + replacement = "vf"; + return true; + } + } + if ((phoneme == langCode + "q" || phoneme == "q") && HasPhoneme("cl")) { replacement = "cl"; return true; } - - // (doesn't work currently) Automatic relaxed consonants: replace "t" or "d" with "dx" if nextNeighbour is a vowel - if ((phoneme == langCode + "t" || phoneme == langCode + "d") && nextNeighbour.HasValue && IsVowel(nextNeighbour.Value)) { - replacement = langCode + "dx"; + if (phoneme == langCode + "q" && !HasPhoneme("cl")) { + replacement = "q"; + return true; + } + if (phoneme == langCode + "q" && !HasPhoneme("cl") && HasPhoneme(langCode + "q")) { + replacement = langCode + "q"; // Keep the language-specific "q" + return true; + } + if (phoneme == "ax" && !HasPhoneme("ax")) { + return true; + } + if (phoneme == langCode + "ax" && !HasPhoneme(langCode + "ax")) { + replacement = langCode + "ah"; // Replace language-specific "ax" with "ah" return true; } - return false; - } - - // Method to check if a phoneme is a vowel - private bool IsVowel(Note note) { - // Retrieve the language code and vowel list - var langCode = GetLangCode(); - string[] baseVowels = GetBaseG2pVowels(); - - // Create the full list of vowel phonemes with language code - string[] vowels = baseVowels.Select(vowel => $"{langCode}/{vowel}").ToArray(); - - // Combine langCode and note.lyric to get the phoneme to check - string phonemeToCheck = $"{langCode}/{note.lyric}"; - - // Log the phoneme being checked and the list of vowels - Log.Debug($"IsVowel: Checking phoneme={phonemeToCheck}"); - Log.Debug($"IsVowel: Available vowels={string.Join(", ", vowels)}"); - // Check if the phonemeToCheck is in the list of vowels - return vowels.Contains(phonemeToCheck); + return false; } + } } diff --git a/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerG2pPhonemizer.cs b/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerG2pPhonemizer.cs index c46c8ddd3..c1648c335 100644 --- a/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerG2pPhonemizer.cs +++ b/OpenUtau.Core/DiffSinger/Phonemizers/DiffSingerG2pPhonemizer.cs @@ -1,10 +1,11 @@ -using Serilog; +using Serilog; using System; using System.Collections.Generic; using System.IO; using System.Linq; using OpenUtau.Api; +using OpenUtau.Classic; namespace OpenUtau.Core.DiffSinger { @@ -35,7 +36,12 @@ public abstract class DiffSingerG2pPhonemizer : DiffSingerBasePhonemizer //vowels and consonants of BaseG2p protected virtual string[] GetBaseG2pVowels()=>new string[]{}; protected virtual string[] GetBaseG2pConsonants()=>new string[]{}; - + + private Dictionary phonemeSymbols = new Dictionary(); + protected bool HasPhoneme(string phoneme) { + return phonemeSymbols.ContainsKey(phoneme); + } + protected override IG2p LoadG2p(string rootPath, bool useLangId = false) { //Each phonemizer has a delicated dictionary name, such as dsdict-en.yaml, dsdict-ru.yaml. //If this dictionary exists, load it. @@ -54,6 +60,13 @@ protected override IG2p LoadG2p(string rootPath, bool useLangId = false) { var dictData = Yaml.DefaultDeserializer.Deserialize(dictText); g2pBuilder.Load(dictData); replacements = dictData.replacementsDict(); + // Collect all symbols from the dictionary and add them to phonemeSymbols + if (dictData.symbols != null) { + foreach (var symbol in dictData.symbols) { + phonemeSymbols[symbol.symbol.Trim()] = true; + } + } + Log.Error("Loaded symbols: " + string.Join(", ", phonemeSymbols.Keys)); } catch (Exception e) { Log.Error(e, $"Failed to load {dictionaryPath}"); } @@ -70,7 +83,6 @@ protected override IG2p LoadG2p(string rootPath, bool useLangId = false) { if(baseG2p == null){ return new G2pFallbacks(g2ps.ToArray()); } - var phonemeSymbols = new Dictionary(); foreach(var v in GetBaseG2pVowels()){ phonemeSymbols[v]=true; }