[{"data":1,"prerenderedAt":332},["ShallowReactive",2],{"content-query-6YegbPk3H3":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":8,"date":9,"cover":10,"type":11,"category":12,"body":13,"_type":326,"_id":327,"_source":328,"_file":329,"_stem":330,"_extension":331},"/technology-blogs/zh/1858","zh",false,"","【MindSpore易点通】深度学习系列-循环神经网络上篇","2022-09-22","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2022/09/29/943c11cdb3114136b2eea97c5fbcba28.png","technology-blogs","基础知识",{"type":14,"children":15,"toc":319},"root",[16,24,30,41,46,55,60,65,70,75,84,89,94,99,104,123,128,135,140,147,152,157,162,171,176,184,189,194,199,204,212,238,243,251,259,264,269,277,289,301,306,314],{"type":17,"tag":18,"props":19,"children":21},"element","h1",{"id":20},"mindspore易点通深度学习系列-循环神经网络上篇",[22],{"type":23,"value":8},"text",{"type":17,"tag":25,"props":26,"children":27},"p",{},[28],{"type":23,"value":29},"学完了卷积神经网络，循环神经网络可不能落下！循环神经网络（RNN）可是在语音识别、自然语言处理等其他领域中引起了变革！",{"type":17,"tag":31,"props":32,"children":34},"h2",{"id":33},"_11-应用场景",[35],{"type":17,"tag":36,"props":37,"children":38},"strong",{},[39],{"type":23,"value":40},"1.1 应用场景",{"type":17,"tag":25,"props":42,"children":43},{},[44],{"type":23,"value":45},"循环神经网络（RNN）其实就是序列模型，我们先来看看其应用场景。",{"type":17,"tag":25,"props":47,"children":48},{},[49],{"type":17,"tag":50,"props":51,"children":54},"img",{"alt":52,"src":53},"cke_4565.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916093659.63213660214566075491126297233180:50530928082110:2400:C042318EBBE0B70AA28203B35097A580D1E57415D9485407F4922FEDF99024EE.png",[],{"type":17,"tag":25,"props":56,"children":57},{},[58],{"type":23,"value":59},"在语音识别时，给定了一个输入音频片段X ，并要求输出对应的文字记录Y 。这里的输入和输出数据都是序列模型，输入X是一个按时播放的音频片段，输出Y是一系列文字。",{"type":17,"tag":25,"props":61,"children":62},{},[63],{"type":23,"value":64},"音乐生成问题也是一样，输出数据Y是序列，而输入数据X可以是空集，也可以是个单一的整数（代表音符）。",{"type":17,"tag":25,"props":66,"children":67},{},[68],{"type":23,"value":69},"而系列模型在DNA序列分析中也十分有用，DNA可以用A、C、G、T四个字母来表示。所以给定一段DNA序列，你能够标记出哪部分是匹配某种蛋白质的吗？",{"type":17,"tag":25,"props":71,"children":72},{},[73],{"type":23,"value":74},"以上所有类似问题都可以被称作使用标签数据(X，Y)作为训练集的监督学习，输入数据X或者输出数据Y是序列，即使两者都是序列也有数据长度不同的问题。",{"type":17,"tag":31,"props":76,"children":78},{"id":77},"_12-模型构建",[79],{"type":17,"tag":36,"props":80,"children":81},{},[82],{"type":23,"value":83},"1.2 模型构建",{"type":17,"tag":25,"props":85,"children":86},{},[87],{"type":23,"value":88},"比如建立一个序列模型，它的输入语句是这样的：“Sam Li and Tom date on Tuesday.”。然后模型是可以自动识别句中人名位置的命名实体识别模型，可以用来查找不同类型的文本中的人名、公司名、时间、地点、国家名和货币名等等。",{"type":17,"tag":25,"props":90,"children":91},{},[92],{"type":23,"value":93},"假定输入数据x，序列模型的输出y，使得输入的每个单词都对应一个输出值，同时y还需要表明输入的单词是否是人名的一部分。",{"type":17,"tag":25,"props":95,"children":96},{},[97],{"type":23,"value":98},"首先输入语句是7个单词组成的时序序列，所以最终会有7个特征集x：x\u003C1>，x\u003C2>，...，x\u003C7>，同时可以索引其序列中的位置。Tx表示输出序列的长度，这里Tx=7。",{"type":17,"tag":25,"props":100,"children":101},{},[102],{"type":23,"value":103},"同理，输出数据也是一样，分别对应y\u003C1>，y\u003C2>，...，y\u003C7>，Ty表示输出序列的长度。",{"type":17,"tag":25,"props":105,"children":106},{},[107,109,114,116,121],{"type":23,"value":108},"因此总结来说，训练样本i的序列中第t个元素用x(i)表示，",{"type":17,"tag":50,"props":110,"children":113},{"alt":111,"src":112},"cke_28207.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916091746.16308699143999421005080893871520:50530928082110:2400:30ECCBC26C1AB57307643B65C9697A6F28F2A99621B425B24EC9401269D05582.png",[],{"type":23,"value":115}," 代表第i个训练样本的输入序列长度，y(i)表示第i个训练样本中第t个元素，",{"type":17,"tag":50,"props":117,"children":120},{"alt":118,"src":119},"cke_44502.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916091815.99087174514344720449975116184755:50530928082110:2400:7D36B524CD118B05D05E323032E11580A2E2902B798587589431888FF88FA432.png",[],{"type":23,"value":122},"就是第i个训练样本的输出序列的长度。",{"type":17,"tag":25,"props":124,"children":125},{},[126],{"type":23,"value":127},"那么问题来了，首先我们需要准备一个比较大的词典库，可能该库里的第一个单词是a，and出现在第367个位置上，Sam是在7459这个位置，Tom则在8674。",{"type":17,"tag":25,"props":129,"children":130},{},[131],{"type":17,"tag":50,"props":132,"children":134},{"alt":7,"src":133},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2022/09/29/5ef2ca77756540d9991942f2ba529450.png",[],{"type":17,"tag":25,"props":136,"children":137},{},[138],{"type":23,"value":139},"那么我们就可以在这个词典库的基础上遍历训练集。",{"type":17,"tag":25,"props":141,"children":142},{},[143],{"type":17,"tag":50,"props":144,"children":146},{"alt":7,"src":145},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2022/09/29/38f5697cff9240e09cb2e589c68f5beb.png",[],{"type":17,"tag":25,"props":148,"children":149},{},[150],{"type":23,"value":151},"那么也就是说，Sam由x\u003C1>表示，其是一个第7459行是1，其余值都是0的向量；Li由x\u003C2>表示，其是一个第2420行是1，其余值都是0的向量。",{"type":17,"tag":25,"props":153,"children":154},{},[155],{"type":23,"value":156},"通常我们称这种x指代句子里的任意词为one-hot向量，只有一个值是1，其余值都是0，所以整句话中我们会有7个one-hot向量，用序列模型在X和Y目标输出之间学习建立一个映射关系。",{"type":17,"tag":25,"props":158,"children":159},{},[160],{"type":23,"value":161},"PS:如果遇到了一个在你词表中的单词，可以创建一个Unknow Word的伪造单词，用作为标记。",{"type":17,"tag":31,"props":163,"children":165},{"id":164},"_13-模型解释",[166],{"type":17,"tag":36,"props":167,"children":168},{},[169],{"type":23,"value":170},"1.3 模型解释",{"type":17,"tag":25,"props":172,"children":173},{},[174],{"type":23,"value":175},"通常情况下，我们会首先选取标准的神经网络，输入7个one-hot向量，经过一些隐藏层，最终会输出7个值为0或1的项，表明每个输入单词是否是人名的一部分。",{"type":17,"tag":25,"props":177,"children":178},{},[179],{"type":17,"tag":50,"props":180,"children":183},{"alt":181,"src":182},"cke_52902.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916093942.53265591691462530967368092288222:50530928082110:2400:5C01A234B2B44D1131D2A671A009C1FBEDA72C9CE56CBAA3453DF6743C4014F4.png",[],{"type":17,"tag":25,"props":185,"children":186},{},[187],{"type":23,"value":188},"但最后我们总会遇到这样的问题：",{"type":17,"tag":25,"props":190,"children":191},{},[192],{"type":23,"value":193},"1.输入和输出数据的长度并不完全一致，即使采用填充（pad）或零填充（zero pad）使每个输入语句都达到最大长度，但最后的表达式会很奇怪。",{"type":17,"tag":25,"props":195,"children":196},{},[197],{"type":23,"value":198},"2.简单的神经网络并不会共享从文本的不同位置上学到的特征。因为我们希望，如果首次学习的时候我们已经知道了Tom是人名，那么当Tom出现在其他位置时，其并不能够自动识别，因此也不能够减少模型中参数的数量。",{"type":17,"tag":25,"props":200,"children":201},{},[202],{"type":23,"value":203},"那么循环神经网络为啥会比普通的神经网络更加出众呢？",{"type":17,"tag":25,"props":205,"children":206},{},[207],{"type":17,"tag":50,"props":208,"children":211},{"alt":209,"src":210},"cke_12460.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916093737.89557219297045742146687519689532:50530928082110:2400:BE8954DA3952CAABEAFC57CDAFAB1BF9E8C1BD61DB527A5C670F0FFFC963EF40.png",[],{"type":17,"tag":25,"props":213,"children":214},{},[215,217,222,224,229,231,236],{"type":23,"value":216},"以从左到右的顺序读这个句子，第一个单词就是x\u003C1>，将其输入一个神经网络层，同时尝试预测输出，判断这是否为人名;下面精彩的部分来啦，循环神经网络中当它读到句中的第二个单词时，假设是x\u003C2>，它不是仅用x\u003C2>就预测出",{"type":17,"tag":50,"props":218,"children":221},{"alt":219,"src":220},"cke_88636.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916092003.11870890475371797715295850605639:50530928082110:2400:908986DF8BC4E6381D5AF5919430FA4784D7DB59529B0367A8742113E9314F01.png",[],{"type":23,"value":223}," ，还会输入一些来自时间步1的信息，那么时间步1的激活值就会传递到时间步2。在下一个时间步，循环神经网络输入了x\u003C3>，尝试预测输出了",{"type":17,"tag":50,"props":225,"children":228},{"alt":226,"src":227},"cke_100161.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916092023.49262290313417228607050307618009:50530928082110:2400:741CD05E5E7361C56580ED366D6A965FC9A1FBE7B902BD819BC0092E49432390.png",[],{"type":23,"value":230}," ，...，一直到最后一个时间步，输入x，然后输出",{"type":17,"tag":50,"props":232,"children":235},{"alt":233,"src":234},"cke_113130.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916092056.75577343087760591693711002686262:50530928082110:2400:16F578B15278265D719398C6480AC82294D6B75637A6607594D6BAFB39E03292.png",[],{"type":23,"value":237}," 。在每一个时间步中，循环神经网络会传递一个激活值到下一个时间步中用于计算，同时还需要使用零向量作为零时刻的伪激活值输入神经网络",{"type":17,"tag":25,"props":239,"children":240},{},[241],{"type":23,"value":242},"另外循环神经网络是从左向右扫描数据，同时每个时间步的参数也是共享的，用Wax来表示从x\u003C1>到隐藏层的连接的一系列参数，每个时间步使用的都是相同的Wax参数，而激活值是由参数Waa决定的，输出结果由Way决定。",{"type":17,"tag":25,"props":244,"children":245},{},[246],{"type":17,"tag":50,"props":247,"children":250},{"alt":248,"src":249},"cke_27232.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916093843.84893534544618606112501472974040:50530928082110:2400:92D16D02D7B22D2565626A0A2E1E8671068013D0C8556DDE2347394C11DEB8C5.png",[],{"type":17,"tag":25,"props":252,"children":253},{},[254],{"type":17,"tag":50,"props":255,"children":258},{"alt":256,"src":257},"cke_37155.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916093900.43400273951846119524172172268350:50530928082110:2400:92D5F1DA2140B51A9928FC002A6C333B13A5FB0099CACA0748E8DC5DB9738EBD.png",[],{"type":17,"tag":25,"props":260,"children":261},{},[262],{"type":23,"value":263},"先输入零向量a\u003C0>，接着进行前向传播过程，计算激活值a\u003C1>，然后再计算y\u003C1>。",{"type":17,"tag":25,"props":265,"children":266},{},[267],{"type":23,"value":268},"更普遍来说，在t时刻:",{"type":17,"tag":25,"props":270,"children":271},{},[272],{"type":17,"tag":50,"props":273,"children":276},{"alt":274,"src":275},"cke_44680.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916093911.95746692558489169073442360776579:50530928082110:2400:C0446335D6152D998A6DA7F9844F8032058A731DDE2BF7AEDF3006F261CAF981.png",[],{"type":17,"tag":25,"props":278,"children":279},{},[280,282,287],{"type":23,"value":281},"为了更加简化一点，定义Wa:",{"type":17,"tag":50,"props":283,"children":286},{"alt":284,"src":285},"cke_162501.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916092243.94741896747961097394280098275838:50530928082110:2400:B5AD111D34C34C24394C6A04DB64188576D0EFF78386F8C1B57B7FF08D0CCD23.png",[],{"type":23,"value":288}," ，假设a是100维的，x是10,000维的，那么Waa是（100，100）维的矩阵，Wax是（100，10000）维，Wa为（100，10100）。",{"type":17,"tag":25,"props":290,"children":291},{},[292,294,299],{"type":23,"value":293},"同样，假定",{"type":17,"tag":50,"props":295,"children":298},{"alt":296,"src":297},"cke_174253.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916092305.23383250714788002915043609549209:50530928082110:2400:C3EFFEEFDD3A7ABF7F5F34F699410D732A4DA5C240231DA405C2EE07FB97960C.png",[],{"type":23,"value":300}," ，Wy表明它是计算y类型的量的权重矩阵，而Wa和ba则表示这些参数是用来计算激活值的。",{"type":17,"tag":25,"props":302,"children":303},{},[304],{"type":23,"value":305},"RNN前向传播示意图：",{"type":17,"tag":25,"props":307,"children":308},{},[309],{"type":17,"tag":50,"props":310,"children":313},{"alt":311,"src":312},"cke_179916.png","https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/5e4/e02/8f7/550440a1fe5e4e028f77e5cf18005adc.20220916092325.90334123402311227994483743536830:50530928082110:2400:38009C1E4B54EDB6AB7E01EF80999248C080797A5B1B5496004A458DE422B20B.png",[],{"type":17,"tag":25,"props":315,"children":316},{},[317],{"type":23,"value":318},"好啦，今天的介绍就到这里啦，下期我们一起学习反向传播，以及如何能够用RNN进行学习！",{"title":7,"searchDepth":320,"depth":320,"links":321},4,[322,324,325],{"id":33,"depth":323,"text":40},2,{"id":77,"depth":323,"text":83},{"id":164,"depth":323,"text":170},"markdown","content:technology-blogs:zh:1858.md","content","technology-blogs/zh/1858.md","technology-blogs/zh/1858","md",1776506116210]