[{"data":1,"prerenderedAt":388},["ShallowReactive",2],{"content-query-Xg8LliwnD6":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":382,"_id":383,"_source":384,"_file":385,"_stem":386,"_extension":387},"/technology-blogs/zh/3240","zh",false,"","声音克隆术：用GPT-SoVITS-MindSpore打造你的数字声音分身","作者：李子洲，昇思MindSpore开发者，西安科技大学大三在读","2024-07-22","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/07/26/a659773803e846c3a7a542e3b4fa1020.png","technology-blogs","调试调优",{"type":15,"children":16,"toc":374},"root",[17,25,31,36,41,59,68,73,81,86,94,99,107,115,123,128,136,141,149,157,167,174,182,189,197,204,211,222,227,232,240,248,258,263,272,277,286,291,300,305,310,318,323,330,335,340,345,350,358,363],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"声音克隆术用gpt-sovits-mindspore打造你的数字声音分身",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":28},"p",{},[29],{"type":24,"value":30},"作者李子洲，昇思MindSpore开发者，西安科技大学大三在读",{"type":18,"tag":26,"props":32,"children":33},{},[34],{"type":24,"value":35},"你是否想象过拥有一个和你声音一模一样的虚拟分身？现在，通过基于GPT-SoVITS的语音生成技术，这一幻想可以成为现实！",{"type":18,"tag":26,"props":37,"children":38},{},[39],{"type":24,"value":40},"昇思MindSpore已支持音色克隆项目 GPT-SoVITS，使用当下流行的影视角色、动漫人物的音色，即可花样定制各类“声音克隆”，让我们一起探索这项令人兴奋的技术，看看如何基于MindSpore NLP + GPT-SoVITS一步步创造出专属于你的数字声音分身。",{"type":18,"tag":42,"props":43,"children":44},"ul",{},[45],{"type":18,"tag":46,"props":47,"children":48},"li",{},[49,51],{"type":24,"value":50},"项目链接：",{"type":18,"tag":52,"props":53,"children":57},"a",{"href":54,"rel":55},"https://github.com/ultranationalism/GPT-SoVITS-mindspore",[56],"nofollow",[58],{"type":24,"value":54},{"type":18,"tag":26,"props":60,"children":61},{},[62],{"type":18,"tag":63,"props":64,"children":65},"strong",{},[66],{"type":24,"value":67},"0****1",{"type":18,"tag":26,"props":69,"children":70},{},[71],{"type":24,"value":72},"零样本文本到语音（TTS）： 输入 5 秒的声音样本，即刻体验文本到语音转换。",{"type":18,"tag":26,"props":74,"children":75},{},[76],{"type":18,"tag":63,"props":77,"children":78},{},[79],{"type":24,"value":80},"0****2",{"type":18,"tag":26,"props":82,"children":83},{},[84],{"type":24,"value":85},"少样本 TTS： 仅需 1 分钟的训练数据即可微调模型，提升声音相似度和真实感。",{"type":18,"tag":26,"props":87,"children":88},{},[89],{"type":18,"tag":63,"props":90,"children":91},{},[92],{"type":24,"value":93},"0****3",{"type":18,"tag":26,"props":95,"children":96},{},[97],{"type":24,"value":98},"跨语言支持： 支持与训练数据集不同语言的推理，目前支持英语、日语和中文。",{"type":18,"tag":26,"props":100,"children":101},{},[102],{"type":18,"tag":63,"props":103,"children":104},{},[105],{"type":24,"value":106},"训练“声音克隆”——",{"type":18,"tag":26,"props":108,"children":109},{},[110],{"type":18,"tag":63,"props":111,"children":112},{},[113],{"type":24,"value":114},"基于MindSpore NLP + GPT-SoVITS生成语音",{"type":18,"tag":26,"props":116,"children":117},{},[118],{"type":18,"tag":63,"props":119,"children":120},{},[121],{"type":24,"value":122},"1. 首先，需要进入项目文件夹",{"type":18,"tag":26,"props":124,"children":125},{},[126],{"type":24,"value":127},"cd GPT-SoVITS-mindspore。",{"type":18,"tag":26,"props":129,"children":130},{},[131],{"type":18,"tag":63,"props":132,"children":133},{},[134],{"type":24,"value":135},"2. 打开webui",{"type":18,"tag":26,"props":137,"children":138},{},[139],{"type":24,"value":140},"launch_webui.sh，弹出这段代码后，说明webui已经打开，可通过127.0.0.1:9872进入webui。",{"type":18,"tag":26,"props":142,"children":143},{},[144],{"type":18,"tag":145,"props":146,"children":148},"img",{"alt":7,"src":147},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/07/26/29bccb33e4004160b76e75fc34c67c0c.png",[],{"type":18,"tag":26,"props":150,"children":151},{},[152],{"type":18,"tag":63,"props":153,"children":154},{},[155],{"type":24,"value":156},"3. 选择GPT模型和SoVITS模型，基础的模型可在下方链接下载：",{"type":18,"tag":26,"props":158,"children":159},{},[160],{"type":18,"tag":52,"props":161,"children":164},{"href":162,"rel":163},"https://modelscope.cn/models/ultranationalism/GPT-SoVITS-mindspore/files%E3%80%82",[56],[165],{"type":24,"value":166},"https://modelscope.cn/models/ultranationalism/GPT-SoVITS-mindspore/files。",{"type":18,"tag":26,"props":168,"children":169},{},[170],{"type":18,"tag":145,"props":171,"children":173},{"alt":7,"src":172},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/07/26/d7bf86aa03c94b00a909754780cede76.png",[],{"type":18,"tag":26,"props":175,"children":176},{},[177],{"type":18,"tag":63,"props":178,"children":179},{},[180],{"type":24,"value":181},"4. 上传参考语音并填入参考音频的文本",{"type":18,"tag":26,"props":183,"children":184},{},[185],{"type":18,"tag":145,"props":186,"children":188},{"alt":7,"src":187},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/07/26/de96b9b4307a432fbf95cdcafb906e6f.png",[],{"type":18,"tag":26,"props":190,"children":191},{},[192],{"type":18,"tag":63,"props":193,"children":194},{},[195],{"type":24,"value":196},"5. 输入需要合成的文本，并设置GPT采样参数，就可以在右侧看到结果",{"type":18,"tag":26,"props":198,"children":199},{},[200],{"type":18,"tag":145,"props":201,"children":203},{"alt":7,"src":202},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/07/26/979a0015c3b1422b875ec1ca748dac88.png",[],{"type":18,"tag":26,"props":205,"children":206},{},[207],{"type":18,"tag":145,"props":208,"children":210},{"alt":7,"src":209},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/07/26/64f33cc4e1354cc6bf3c952eff226a10.png",[],{"type":18,"tag":42,"props":212,"children":213},{},[214],{"type":18,"tag":46,"props":215,"children":216},{},[217],{"type":18,"tag":63,"props":218,"children":219},{},[220],{"type":24,"value":221},"GPT采样参数说明",{"type":18,"tag":26,"props":223,"children":224},{},[225],{"type":24,"value":226},"top_k、top_p、temperature这三个值都是用来控制采样的。在推理的时候要挑出一个最好的token，但机器并不知道哪个是最好的。于是先按照top_k挑出前几个token，top_p在top_k的基础上筛选token。最后temperature控制随机性输出。",{"type":18,"tag":26,"props":228,"children":229},{},[230],{"type":24,"value":231},"比如总共有100个token，top_k设置5，top_p设置0.6，temperature设置为0.5。那么就会从100个token中先挑出5个概率最大的token，这五个token的概率分别是（0.3，0.3，0.2，0.2，0.1），那么再挑出累加概率不超过0.6的token（0.3和0.3），再从这两个token中随机挑出一个token输出，其中前一个token被挑选到的几率更大。以此类推。",{"type":18,"tag":26,"props":233,"children":234},{},[235],{"type":18,"tag":63,"props":236,"children":237},{},[238],{"type":24,"value":239},"捕捉声音的灵魂——",{"type":18,"tag":26,"props":241,"children":242},{},[243],{"type":18,"tag":63,"props":244,"children":245},{},[246],{"type":24,"value":247},"GPT-SoVITS TTS的原理",{"type":18,"tag":249,"props":250,"children":252},"h3",{"id":251},"_1-特征提取",[253],{"type":18,"tag":63,"props":254,"children":255},{},[256],{"type":24,"value":257},"1. 特征提取",{"type":18,"tag":26,"props":259,"children":260},{},[261],{"type":24,"value":262},"Hubert和RVQ将音频转化为包含音色的语义token，然后使用类GPT模型将他补全。",{"type":18,"tag":249,"props":264,"children":266},{"id":265},"_2-类gpt模型",[267],{"type":18,"tag":63,"props":268,"children":269},{},[270],{"type":24,"value":271},"2. 类GPT模型",{"type":18,"tag":26,"props":273,"children":274},{},[275],{"type":24,"value":276},"类GPT模型的自回归特性，我们可以在类GPT模型推理的时候添加一段参考音频，推理时的token始终会继承一些参考音频的音色。这样，补全的包含声音的token就由音色emb和参考音频共同决定。",{"type":18,"tag":249,"props":278,"children":280},{"id":279},"_3-音频重建",[281],{"type":18,"tag":63,"props":282,"children":283},{},[284],{"type":24,"value":285},"3. 音频重建",{"type":18,"tag":26,"props":287,"children":288},{},[289],{"type":24,"value":290},"最后，使用VITS模型对生成的语音token进行解码，生成最终的音频。VITS模型能够有效地重建高质量的语音，保留文本的语义信息和目标音色特征。因为token承担了一部分音色引导功能，VITS模型重建目标音色的压力就小很多。",{"type":18,"tag":249,"props":292,"children":294},{"id":293},"_4-基线",[295],{"type":18,"tag":63,"props":296,"children":297},{},[298],{"type":24,"value":299},"4. 基线",{"type":18,"tag":26,"props":301,"children":302},{},[303],{"type":24,"value":304},"SoVITS大模型需要学习万物音色，从量变到质变，才能通过音色emb进行引导，找到相似音色并重建目标音色。",{"type":18,"tag":26,"props":306,"children":307},{},[308],{"type":24,"value":309},"GPT-SoVITS: token承担一部分音色引导功能，SoVITS大模型的训练集音色覆盖压力大大降低，使5秒音色克隆(变声)具备可行性。",{"type":18,"tag":26,"props":311,"children":312},{},[313],{"type":18,"tag":63,"props":314,"children":315},{},[316],{"type":24,"value":317},"写到最后",{"type":18,"tag":26,"props":319,"children":320},{},[321],{"type":24,"value":322},"下面将简单介绍GPT-SoVITS的动机与他的由来。我们从语音转换（变声）领域最有名，且上了2023年时代周刊前200大发明的SoVITS讲起。",{"type":18,"tag":26,"props":324,"children":325},{},[326],{"type":18,"tag":145,"props":327,"children":329},{"alt":7,"src":328},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/07/26/bb98b8d5869c49bfa6d3b146ee1d0add.png",[],{"type":18,"tag":26,"props":331,"children":332},{},[333],{"type":24,"value":334},"SoVITS的原理可以很简单地抽象为上图右边的编码器解码器结构。在重建的过程中模型将训练集音色学习进它的参数中，因此输入其他音色的语音经过编码后，VITS模型也能重建出学习过的音色的语音。在这个叙述逻辑中，问题最大的一点在于编码器去音色这一步。事实上，目前就没有能完全去除音色的特征，只有去音色相对强的特征,这也是为什么我们最后会使用ContentVec代替SoVITS名称中的SoftVC作为编码器的原因。",{"type":18,"tag":26,"props":336,"children":337},{},[338],{"type":24,"value":339},"漏音色的意思是，你转换转了个“寂寞”，不太是你想要的那个音色，除了AI的噱头外就没什么太大用途了。但是我们反过来想，我们完全可以用这个泄露音色的语义特征作为音色的引导，来引导目标的音色。于是，第一批进入观众席的音色漏麻的CN_Hubert（原始Hubert使用wenet中文数据训练的参数）重新回到了赛场，保留了丰富的音色信息反而成为了他的优势。我们使用Hubert和RVQ将音频转化为包含音色的语义token，然后使用类GPT模型将他补全。因为token已经给音色划分了一个范围，因此给VITS重建目标音色的压力小了很多baseline：SoVITS大模型需要学习万物音色，从量变到质变，才能通过音色emb进行引导，找到相似音色并重建目标音色。GPT-SoVITS token承担一部分音色引导功能，SoVITS大模型的训练集音色覆盖压力大大降低，使5秒音色克隆(变声)具备可行性。",{"type":18,"tag":26,"props":341,"children":342},{},[343],{"type":24,"value":344},"由于类GPT模型的自回归特性，推理时后面的token始终会继承一些参考音频的音色。",{"type":18,"tag":26,"props":346,"children":347},{},[348],{"type":24,"value":349},"又由于token使用的特征漏音色比condition语义特征更大，因此GPT补全的token更多的是包含参考音频的目标音色。在Hubert分担一部分音色压力后，剩下的VITS的重建目标音色的压力就小很多了，这使得5秒音色克隆(变声)终于具备了可行性。我们也可以将GPT模型的condition变更为文本。",{"type":18,"tag":26,"props":351,"children":352},{},[353],{"type":18,"tag":63,"props":354,"children":355},{},[356],{"type":24,"value":357},"更多推荐",{"type":18,"tag":26,"props":359,"children":360},{},[361],{"type":24,"value":362},"昇思MindSpore2.3版本已经正式上线，增强动/静态图与大模型能力，欢迎体验！详细特性介绍请戳下文了解。",{"type":18,"tag":26,"props":364,"children":365},{},[366,368],{"type":24,"value":367},"安装信息：",{"type":18,"tag":52,"props":369,"children":372},{"href":370,"rel":371},"https://www.mindspore.cn/install",[56],[373],{"type":24,"value":370},{"title":7,"searchDepth":375,"depth":375,"links":376},4,[377,379,380,381],{"id":251,"depth":378,"text":257},3,{"id":265,"depth":378,"text":271},{"id":279,"depth":378,"text":285},{"id":293,"depth":378,"text":299},"markdown","content:technology-blogs:zh:3240.md","content","technology-blogs/zh/3240.md","technology-blogs/zh/3240","md",1776506127659]