[{"data":1,"prerenderedAt":922},["ShallowReactive",2],{"content-query-35hlcDj72j":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"body":13,"_type":916,"_id":917,"_source":918,"_file":919,"_stem":920,"_extension":921},"/technology-blogs/zh/405","zh",false,"","AI框架如何帮助开发者提升精度调优效率","针对AI框架(MindSpore)如何帮助模型开发者提升精度调优效率的问题，作者总结了一些思路供大家思考和讨论。","2021-03-03","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/03/03/bdc69766d4594847b8280f3662ed4ab8.png","technology-blogs",{"type":14,"children":15,"toc":901},"root",[16,24,30,43,54,59,65,70,79,104,112,135,143,151,165,173,191,214,222,230,238,256,261,266,271,276,281,286,291,296,301,309,314,319,327,332,337,342,347,352,357,375,380,385,418,423,431,436,441,446,451,499,504,511,516,523,528,533,538,543,548,601,606,613,618,623,630,635,648,656,661,679,697,705,710,715,733,738,743,756,761,766,774,779,802,807,813,818,841,847,852,865,870,875,888],{"type":17,"tag":18,"props":19,"children":21},"element","h1",{"id":20},"ai框架如何帮助开发者提升精度调优效率",[22],{"type":23,"value":8},"text",{"type":17,"tag":25,"props":26,"children":27},"p",{},[28],{"type":23,"value":29},"作者:金雪锋",{"type":17,"tag":25,"props":31,"children":32},{},[33,35],{"type":23,"value":34},"文章链接：",{"type":17,"tag":36,"props":37,"children":41},"a",{"href":38,"rel":39},"https://zhuanlan.zhihu.com/p/353715732",[40],"nofollow",[42],{"type":23,"value":38},{"type":17,"tag":25,"props":44,"children":45},{},[46,48],{"type":23,"value":47},"作者主页：",{"type":17,"tag":36,"props":49,"children":52},{"href":50,"rel":51},"https://www.zhihu.com/people/jin-xue-feng",[40],[53],{"type":23,"value":50},{"type":17,"tag":25,"props":55,"children":56},{},[57],{"type":23,"value":58},"最近在考虑AI框架(MindSpore)如何帮助模型开发者提升精度调优效率的问题，总结了一些思路供参考。",{"type":17,"tag":60,"props":61,"children":63},"h2",{"id":62},"模型准确率不高的常见原因",[64],{"type":23,"value":62},{"type":17,"tag":25,"props":66,"children":67},{},[68],{"type":23,"value":69},"模型的准确率不高，模型的metrics达不到预期，常见的原因有以下几方面：",{"type":17,"tag":25,"props":71,"children":72},{},[73],{"type":17,"tag":74,"props":75,"children":76},"strong",{},[77],{"type":23,"value":78},"1、数据集问题",{"type":17,"tag":80,"props":81,"children":82},"ul",{},[83,89,94,99],{"type":17,"tag":84,"props":85,"children":86},"li",{},[87],{"type":23,"value":88},"数据集中缺失值过多",{"type":17,"tag":84,"props":90,"children":91},{},[92],{"type":23,"value":93},"数据集每个类别的样本数目不均衡",{"type":17,"tag":84,"props":95,"children":96},{},[97],{"type":23,"value":98},"数据集中存在异常值",{"type":17,"tag":84,"props":100,"children":101},{},[102],{"type":23,"value":103},"数据集中的数据对预测结果帮助不大（例如使用年龄预测性别）",{"type":17,"tag":25,"props":105,"children":106},{},[107],{"type":17,"tag":74,"props":108,"children":109},{},[110],{"type":23,"value":111},"2、数据处理算法设计和实现问题",{"type":17,"tag":80,"props":113,"children":114},{},[115,120,125,130],{"type":17,"tag":84,"props":116,"children":117},{},[118],{"type":23,"value":119},"数据处理参数有误",{"type":17,"tag":84,"props":121,"children":122},{},[123],{"type":23,"value":124},"未对数据进行归一化",{"type":17,"tag":84,"props":126,"children":127},{},[128],{"type":23,"value":129},"特征提取算法（如果使用了）存在错误",{"type":17,"tag":84,"props":131,"children":132},{},[133],{"type":23,"value":134},"train和validation数据处理方式不一致",{"type":17,"tag":25,"props":136,"children":137},{},[138],{"type":17,"tag":74,"props":139,"children":140},{},[141],{"type":23,"value":142},"3、算法设计和实现问题",{"type":17,"tag":80,"props":144,"children":145},{},[146],{"type":17,"tag":84,"props":147,"children":148},{},[149],{"type":23,"value":150},"API使用错误",{"type":17,"tag":152,"props":153,"children":154},"ol",{},[155,160],{"type":17,"tag":84,"props":156,"children":157},{},[158],{"type":23,"value":159},"没有遵循深度学习框架约束",{"type":17,"tag":84,"props":161,"children":162},{},[163],{"type":23,"value":164},"算子使用错误",{"type":17,"tag":80,"props":166,"children":167},{},[168],{"type":17,"tag":84,"props":169,"children":170},{},[171],{"type":23,"value":172},"计算图结构错误",{"type":17,"tag":152,"props":174,"children":175},{},[176,181,186],{"type":17,"tag":84,"props":177,"children":178},{},[179],{"type":23,"value":180},"权重共享错误",{"type":17,"tag":84,"props":182,"children":183},{},[184],{"type":23,"value":185},"权重冻结错误（例如忘记冻结应冻结的权重）",{"type":17,"tag":84,"props":187,"children":188},{},[189],{"type":23,"value":190},"节点连接错误（例如应连接在图中的节点未连接）",{"type":17,"tag":80,"props":192,"children":193},{},[194,199,204,209],{"type":17,"tag":84,"props":195,"children":196},{},[197],{"type":23,"value":198},"loss函数错误",{"type":17,"tag":84,"props":200,"children":201},{},[202],{"type":23,"value":203},"优化器错误",{"type":17,"tag":84,"props":205,"children":206},{},[207],{"type":23,"value":208},"权重初始值错误",{"type":17,"tag":84,"props":210,"children":211},{},[212],{"type":23,"value":213},"算法本身设计有缺陷导致精度无法达到预期。",{"type":17,"tag":25,"props":215,"children":216},{},[217],{"type":17,"tag":74,"props":218,"children":219},{},[220],{"type":23,"value":221},"4、超参设置问题",{"type":17,"tag":25,"props":223,"children":224},{},[225],{"type":17,"tag":74,"props":226,"children":227},{},[228],{"type":23,"value":229},"5、普通python编程错误",{"type":17,"tag":25,"props":231,"children":232},{},[233],{"type":17,"tag":74,"props":234,"children":235},{},[236],{"type":23,"value":237},"6、环境问题",{"type":17,"tag":80,"props":239,"children":240},{},[241,246,251],{"type":17,"tag":84,"props":242,"children":243},{},[244],{"type":23,"value":245},"依赖软件问题",{"type":17,"tag":84,"props":247,"children":248},{},[249],{"type":23,"value":250},"环境变量配置问题",{"type":17,"tag":84,"props":252,"children":253},{},[254],{"type":23,"value":255},"云上环境问题",{"type":17,"tag":60,"props":257,"children":259},{"id":258},"问题分析整体思路",[260],{"type":23,"value":258},{"type":17,"tag":25,"props":262,"children":263},{},[264],{"type":23,"value":265},"分析模型准确率不高的问题，就是要分析数据集、数据处理、算法设计、算法实现等方面是否存在问题。这一过程是有章可循的，建议参考下面五步，逐步定位模型表现问题：",{"type":17,"tag":25,"props":267,"children":268},{},[269],{"type":23,"value":270},"第一步 检查代码和超参",{"type":17,"tag":25,"props":272,"children":273},{},[274],{"type":23,"value":275},"第二步 检查模型结构",{"type":17,"tag":25,"props":277,"children":278},{},[279],{"type":23,"value":280},"第三步 检查输入数据",{"type":17,"tag":25,"props":282,"children":283},{},[284],{"type":23,"value":285},"第四步 检查loss曲线",{"type":17,"tag":25,"props":287,"children":288},{},[289],{"type":23,"value":290},"第五步 检查准确率是否达到预期",{"type":17,"tag":25,"props":292,"children":293},{},[294],{"type":23,"value":295},"代码是准确率问题的重要源头，检查代码重在对脚本和代码做检查，力争在源头发现问题；模型脚本体现了AI模型在AI框架上的表达和映射，检查模型脚本重在检查脚本表达和算法工程师的设计是否一致；有的问题要到动态的训练过程中才会发现，检查输入数据和loss曲线正是将代码和动态训练现象结合进行检查；检查准确率是否达到预期则是对整体分析过程的重新审视。此外，充分准备，熟悉模型也是很重要的。下面将分别介绍这些思路。",{"type":17,"tag":60,"props":297,"children":299},{"id":298},"模型问题分析准备",[300],{"type":23,"value":298},{"type":17,"tag":25,"props":302,"children":303},{},[304],{"type":17,"tag":74,"props":305,"children":306},{},[307],{"type":23,"value":308},"1、回顾算法设计，全面熟悉模型",{"type":17,"tag":25,"props":310,"children":311},{},[312],{"type":23,"value":313},"分析前，要先对算法设计做回顾，确保算法设计明确。如果参考论文实现模型，则应回顾论文中的全部设计细节和超参选择情况；如果参考其它脚本实现模型，则应确保有一个唯一的、精度能够达标的标杆脚本；如果是新开发的算法，也应将重要的设计细节、超参选择、参考论文和脚本明确出来。这些信息是后面检查脚本步骤的重要依据。",{"type":17,"tag":25,"props":315,"children":316},{},[317],{"type":23,"value":318},"分析问题前，还要全面熟悉模型。只有熟悉了模型，才能准确理解各种现象中蕴含的信息，判断是否存在问题，查找问题源头。因此，花时间理解模型算法和结构、理解模型中算子的作用和参数的含义、理解模型所用优化器的特性是很重要的。动手分析问题细节前，建议先带着问题加深对这些模型要素的了解。",{"type":17,"tag":25,"props":320,"children":321},{},[322],{"type":17,"tag":74,"props":323,"children":324},{},[325],{"type":23,"value":326},"2、使用小数据集测试目标脚本能否成功拟合",{"type":17,"tag":25,"props":328,"children":329},{},[330],{"type":23,"value":331},"若模型不能拟合（fit）小数据集，说明模型本身的设计存在问题。",{"type":17,"tag":60,"props":333,"children":335},{"id":334},"第一步-检查代码和超参",[336],{"type":23,"value":270},{"type":17,"tag":25,"props":338,"children":339},{},[340],{"type":23,"value":341},"代码是模型问题的重要源头，超参问题、模型结构问题、数据问题、算法设计和实现问题会体现在脚本中，对脚本做检查是定位模型问题很有效率的手段。",{"type":17,"tag":25,"props":343,"children":344},{},[345],{"type":23,"value":346},"检查代码主要依赖代码走读和单元测试。代码走读建议使用小黄鸭调试法：在代码走读的过程中，耐心地向没有经验的“小黄鸭”解释每一行代码的作用，从而激发灵感，发现代码问题。检查脚本时，要注意检查脚本实现（包括数据处理、模型结构、loss函数、优化器等实现）同设计是否一致，如果参考了其它脚本，要重点检查脚本实现同其它脚本是否一致，所有不一致的地方都应该有充分合理的理由，否则就应修改。",{"type":17,"tag":25,"props":348,"children":349},{},[350],{"type":23,"value":351},"还可以通过编写单元测试的方法检查脚本，将脚本在逻辑上拆分为多个模块或函数，然后编写单元测试，依次对这些模块和函数进行验证。",{"type":17,"tag":25,"props":353,"children":354},{},[355],{"type":23,"value":356},"检查脚本时，也要关注超参的情况，超参问题主要体现为超参取值不合理，例如：",{"type":17,"tag":152,"props":358,"children":359},{},[360,365,370],{"type":17,"tag":84,"props":361,"children":362},{},[363],{"type":23,"value":364},"学习率设置不合理；",{"type":17,"tag":84,"props":366,"children":367},{},[368],{"type":23,"value":369},"loss_scale参数不合理；",{"type":17,"tag":84,"props":371,"children":372},{},[373],{"type":23,"value":374},"权重初始化参数不合理等。",{"type":17,"tag":60,"props":376,"children":378},{"id":377},"第二步-检查模型结构",[379],{"type":23,"value":275},{"type":17,"tag":25,"props":381,"children":382},{},[383],{"type":23,"value":384},"在模型结构方面，常见的问题有：",{"type":17,"tag":152,"props":386,"children":387},{},[388,393,398,403,408,413],{"type":17,"tag":84,"props":389,"children":390},{},[391],{"type":23,"value":392},"算子使用错误（使用的算子不适用于目标场景，如应该使用浮点除，错误地使用了整数除）；",{"type":17,"tag":84,"props":394,"children":395},{},[396],{"type":23,"value":397},"权重共享错误（共享了不应共享的权重）；",{"type":17,"tag":84,"props":399,"children":400},{},[401],{"type":23,"value":402},"权重冻结错误（冻结了不应冻结的权重）；",{"type":17,"tag":84,"props":404,"children":405},{},[406],{"type":23,"value":407},"节点连接错误（应该连接到计算图中的block未连接）；",{"type":17,"tag":84,"props":409,"children":410},{},[411],{"type":23,"value":412},"loss函数错误；",{"type":17,"tag":84,"props":414,"children":415},{},[416],{"type":23,"value":417},"优化器算法错误（如果自行实现了优化器）等。",{"type":17,"tag":25,"props":419,"children":420},{},[421],{"type":23,"value":422},"建议通过检查模型代码的方式对模型结构进行检查。此外，还可以借助各种计算图可视化工具检查计算图。若有标杆脚本，还可以同标杆脚本对照查看计算图，检查当前脚本和标杆脚本的计算图是否存在重要的差异。",{"type":17,"tag":25,"props":424,"children":425},{},[426],{"type":17,"tag":427,"props":428,"children":430},"img",{"alt":7,"src":429},"https://pic2.zhimg.com/80/v2-8afcf5421dc163754cc306f16661207d_720w.jpg",[],{"type":17,"tag":25,"props":432,"children":433},{},[434],{"type":23,"value":435},"通过计算图可视化检查模型结构-来自MindSpore的可视化工具MindInsight",{"type":17,"tag":25,"props":437,"children":438},{},[439],{"type":23,"value":440},"考虑到模型结构一般都很复杂，期望在这一步就能发现所有的模型结构问题是不现实的。只要通过可视化的模型结构加深对计算图的理解，发现明显的结构问题即可。后面的步骤中，发现了更明确的问题现象后，我们还会回到这一步重新检查确认。",{"type":17,"tag":60,"props":442,"children":444},{"id":443},"第三步-检查输入数据",[445],{"type":23,"value":280},{"type":17,"tag":25,"props":447,"children":448},{},[449],{"type":23,"value":450},"通过检查输入模型的数据，可以结合脚本判断数据处理流水线和数据集是否存在问题。输入数据的常见问题有：",{"type":17,"tag":152,"props":452,"children":453},{},[454,459,464,469,474,479,484,489,494],{"type":17,"tag":84,"props":455,"children":456},{},[457],{"type":23,"value":458},"数据缺失值过多；",{"type":17,"tag":84,"props":460,"children":461},{},[462],{"type":23,"value":463},"每个类别中的样本数目不均衡；",{"type":17,"tag":84,"props":465,"children":466},{},[467],{"type":23,"value":468},"数据中存在异常值；",{"type":17,"tag":84,"props":470,"children":471},{},[472],{"type":23,"value":473},"数据标签错误；",{"type":17,"tag":84,"props":475,"children":476},{},[477],{"type":23,"value":478},"训练样本不足；",{"type":17,"tag":84,"props":480,"children":481},{},[482],{"type":23,"value":483},"未对数据进行标准化，输入模型的数据不在正确的范围内；",{"type":17,"tag":84,"props":485,"children":486},{},[487],{"type":23,"value":488},"finetune和pretrain的数据处理方式不同；",{"type":17,"tag":84,"props":490,"children":491},{},[492],{"type":23,"value":493},"训练阶段和推理阶段的数据处理方式不同；",{"type":17,"tag":84,"props":495,"children":496},{},[497],{"type":23,"value":498},"数据处理参数不正确等。",{"type":17,"tag":25,"props":500,"children":501},{},[502],{"type":23,"value":503},"建议借助工具可视化输入数据，若数据明显不符合预期（例如数据被裁剪的范围过大，数据旋转的角度过大等），可以判断输入数据出现了一定的问题。",{"type":17,"tag":25,"props":505,"children":506},{},[507],{"type":17,"tag":427,"props":508,"children":510},{"alt":7,"src":509},"https://pic1.zhimg.com/80/v2-acbd3a8878ff836eb66a666baf56c280_720w.jpg",[],{"type":17,"tag":25,"props":512,"children":513},{},[514],{"type":23,"value":515},"可视化查看输入模型的数据",{"type":17,"tag":25,"props":517,"children":518},{},[519],{"type":17,"tag":427,"props":520,"children":522},{"alt":7,"src":521},"https://pic4.zhimg.com/80/v2-600c90c747e77e73b7968b5e37cd45cf_720w.jpg",[],{"type":17,"tag":25,"props":524,"children":525},{},[526],{"type":23,"value":527},"可视化查看数据处理流水线--来自MindSpore的可视化工具MindInsight",{"type":17,"tag":25,"props":529,"children":530},{},[531],{"type":23,"value":532},"如果有标杆脚本，还可以同标杆脚本对照，检查数据处理流水线输出的数据是否和标杆脚本的数据相同。例如，将数据处理流水线输出的数据保存为npy文件，然后使用numpy.allclose()方法对标杆脚本和当前脚本的数据进行对比。如果发现不同，则数据处理阶段可能存在精度问题。",{"type":17,"tag":25,"props":534,"children":535},{},[536],{"type":23,"value":537},"若数据处理流水线未发现问题，可以手动检查数据集是否存在分类不均衡、标签匹配错误、缺失值过多等问题。",{"type":17,"tag":60,"props":539,"children":541},{"id":540},"第四步-检查loss曲线",[542],{"type":23,"value":285},{"type":17,"tag":25,"props":544,"children":545},{},[546],{"type":23,"value":547},"很多模型问题会在网络训练过程中通过观察loss曲线发现，常见的问题或现象有：",{"type":17,"tag":152,"props":549,"children":550},{},[551,556,561,566,571,576,581,586,591,596],{"type":17,"tag":84,"props":552,"children":553},{},[554],{"type":23,"value":555},"权重初始化不合理（例如初始值为0，初始值范围不合理等）；",{"type":17,"tag":84,"props":557,"children":558},{},[559],{"type":23,"value":560},"权重中存在过大、过小值；",{"type":17,"tag":84,"props":562,"children":563},{},[564],{"type":23,"value":565},"权重变化过大；",{"type":17,"tag":84,"props":567,"children":568},{},[569],{"type":23,"value":570},"权重冻结不正确；",{"type":17,"tag":84,"props":572,"children":573},{},[574],{"type":23,"value":575},"权重共享不正确；",{"type":17,"tag":84,"props":577,"children":578},{},[579],{"type":23,"value":580},"激活值饱和或过弱（例如Sigmoid的输出接近1，Relu的输出全为0）；",{"type":17,"tag":84,"props":582,"children":583},{},[584],{"type":23,"value":585},"梯度爆炸、消失；",{"type":17,"tag":84,"props":587,"children":588},{},[589],{"type":23,"value":590},"训练epoch不足；",{"type":17,"tag":84,"props":592,"children":593},{},[594],{"type":23,"value":595},"算子计算结果存在NAN、INF；",{"type":17,"tag":84,"props":597,"children":598},{},[599],{"type":23,"value":600},"算子计算过程溢出（计算过程中的溢出不一定都是有害的）等。",{"type":17,"tag":25,"props":602,"children":603},{},[604],{"type":23,"value":605},"loss曲线能够反映网络训练的动态趋势，通过观察loss曲线，可以得到模型是否收敛、是否过拟合等信息。loss跑飞和loss收敛慢是模型准确率不高时的主要loss现象。当loss跑飞时，模型的准确率可能只有随机猜测的水平。当loss收敛慢时，若epoch不够多，loss始终也无法收敛到预期值，模型准确率将无法达到预期。",{"type":17,"tag":25,"props":607,"children":608},{},[609],{"type":17,"tag":427,"props":610,"children":612},{"alt":7,"src":611},"https://pic4.zhimg.com/80/v2-60688d209687272346b74cad9346c273_720w.jpg",[],{"type":17,"tag":25,"props":614,"children":615},{},[616],{"type":23,"value":617},"可视化查看loss曲线。图中展示了训练集上的loss随训练迭代数的变化情况。",{"type":17,"tag":25,"props":619,"children":620},{},[621],{"type":23,"value":622},"部分问题（例如错误地冻结了不该冻结的权重）难以直接通过loss曲线观察出来，建议同时查看参数分布图，观察模型参数的变化情况，可以得到权重是否更新过快、过慢，权重分布范围是否正常等信息。",{"type":17,"tag":25,"props":624,"children":625},{},[626],{"type":17,"tag":427,"props":627,"children":629},{"alt":7,"src":628},"https://pic4.zhimg.com/80/v2-dd631f328e67ff5466c2733fb720488f_720w.jpg",[],{"type":17,"tag":25,"props":631,"children":632},{},[633],{"type":23,"value":634},"可视化查看训练过程中的权重变化情况。图中为conv1卷积的权重分布随训练迭代数的变化情况。",{"type":17,"tag":25,"props":636,"children":637},{},[638,640,646],{"type":23,"value":639},"更进一步地，还可以使用深度学习领域的调试器，例如MindSpore框架的调试器（",{"type":17,"tag":36,"props":641,"children":644},{"href":642,"rel":643},"https://www.mindspore.cn/",[40],[645],{"type":23,"value":642},{"type":23,"value":647},"）等工具，对训练现场进行检查，更准确地发现问题。",{"type":17,"tag":25,"props":649,"children":650},{},[651],{"type":17,"tag":74,"props":652,"children":653},{},[654],{"type":23,"value":655},"定位loss跑飞问题",{"type":17,"tag":25,"props":657,"children":658},{},[659],{"type":23,"value":660},"loss跑飞是指loss中出现了NAN、+/-INF或者特别大的值。loss跑飞一般意味着算法设计或实现存在问题。定位思路如下：",{"type":17,"tag":152,"props":662,"children":663},{},[664,669,674],{"type":17,"tag":84,"props":665,"children":666},{},[667],{"type":23,"value":668},"回顾脚本、模型结构和数据：检查超参是否有不合理的特别大/特别小的取值，检查模型结构是否实现正确，特别是检查loss函数是否实现正确，检查输入数据中是否有缺失值、是否有特别大/特别小的取值。",{"type":17,"tag":84,"props":670,"children":671},{},[672],{"type":23,"value":673},"观察参数分布图，检查参数更新是否有明显的异常。若发现参数更新异常，可以结合调试器定位参数更新异常的原因。",{"type":17,"tag":84,"props":675,"children":676},{},[677],{"type":23,"value":678},"使用调试器对训练现场进行检查。",{"type":17,"tag":80,"props":680,"children":681},{},[682,687,692],{"type":17,"tag":84,"props":683,"children":684},{},[685],{"type":23,"value":686},"若loss值出现NAN、+/-INF，可使用“检查张量溢出”条件添加全局监测点，定位首先出现NAN、+/-INF的算子节点，检查算子的输入数据是否会导致计算异常（例如除零）。若是算子输入数据的问题，则可以针对性地加入小数值epsilon避免计算异常。",{"type":17,"tag":84,"props":688,"children":689},{},[690],{"type":23,"value":691},"若loss值出现特别大的值，可使用“检查过大张量”条件添加全局监测点，定位首先出现大值的算子节点，检查算子的输入数据是否会导致计算异常。若输入数据本身存在异常，则可以继续向上查找产生该输入数据的算子，直到定位出具体原因。",{"type":17,"tag":84,"props":693,"children":694},{},[695],{"type":23,"value":696},"若怀疑参数更新、梯度等方面存在异常，可使用“检查权重变化过大”、“检查梯度消失”、“检查梯度过大”等条件设置监测点，定位到异常的权重或梯度，然后结合张量检查视图，逐层向上对可疑的正向算子、反向算子、优化器算子等进行检查。",{"type":17,"tag":25,"props":698,"children":699},{},[700],{"type":17,"tag":74,"props":701,"children":702},{},[703],{"type":23,"value":704},"定位loss收敛慢问题",{"type":17,"tag":25,"props":706,"children":707},{},[708],{"type":23,"value":709},"loss收敛慢是指loss震荡、收敛速度慢，经过很长时间才能达到预期值，或者最终也无法收敛到预期值。相较于loss跑飞，loss收敛慢的数值特征不明显，更难定位。定位思路如下：",{"type":17,"tag":25,"props":711,"children":712},{},[713],{"type":23,"value":714},"1、回顾脚本、模型结构和数据，",{"type":17,"tag":80,"props":716,"children":717},{},[718,723,728],{"type":17,"tag":84,"props":719,"children":720},{},[721],{"type":23,"value":722},"检查超参是否有不合理的特别大/特别小的取值，特别是检查学习率是否设置过小或过大，学习率设置过小会导致收敛速度慢，学习率设置过大会导致loss震荡、不下降；",{"type":17,"tag":84,"props":724,"children":725},{},[726],{"type":23,"value":727},"检查模型结构是否实现正确，特别是检查loss函数、优化器是否实现正确；",{"type":17,"tag":84,"props":729,"children":730},{},[731],{"type":23,"value":732},"检查输入数据的范围是否正常，特别是输入数据的值是否过小",{"type":17,"tag":25,"props":734,"children":735},{},[736],{"type":23,"value":737},"2、观察训练看板中的参数分布图，检查参数更新是否有明显的异常。若发现参数更新异常，可以结合调试器定位参数更新异常的原因。",{"type":17,"tag":25,"props":739,"children":740},{},[741],{"type":23,"value":742},"3、使用调试器模块对训练现场进程检查。",{"type":17,"tag":80,"props":744,"children":745},{},[746,751],{"type":17,"tag":84,"props":747,"children":748},{},[749],{"type":23,"value":750},"可使用“检查权重变化过小”、“检查未变化权重”条件对可训练（未固定）的权重进行监测，检查权重是否变化过小。若发现权重变化过小，可进一步检查学习率取值是否过小、优化器算法是否正确实现、梯度是否消失，并做针对性的修复。",{"type":17,"tag":84,"props":752,"children":753},{},[754],{"type":23,"value":755},"可使用“检查梯度消失”条件对梯度进行监测，检查是否存在梯度消失的现象。若发现梯度消失，可进一步向上检查导致梯度消失的原因。例如，可以通过“检查激活值范围”条件检查是否出现了激活值饱和、Relu输出为0等问题。",{"type":17,"tag":25,"props":757,"children":758},{},[759],{"type":23,"value":760},"4、若认为loss的收敛速度正常，可以尝试增加epoch数目继续训练，看epoch数目增加后，loss能否收敛到预期值。",{"type":17,"tag":60,"props":762,"children":764},{"id":763},"第五步-检查准确率是否达到预期",[765],{"type":23,"value":290},{"type":17,"tag":25,"props":767,"children":768},{},[769],{"type":17,"tag":74,"props":770,"children":771},{},[772],{"type":23,"value":773},"1、检查训练集准确率",{"type":17,"tag":25,"props":775,"children":776},{},[777],{"type":23,"value":778},"若训练集上loss已收敛，但是训练集上的准确率未达到预期，首先应重点检查loss函数是否实现正确，使用前文介绍的方法走读代码、编写单元测试对loss函数进行检查。必要时重新进行前四步，回顾代码、模型结构、输入数据和loss曲线：",{"type":17,"tag":152,"props":780,"children":781},{},[782,787,792,797],{"type":17,"tag":84,"props":783,"children":784},{},[785],{"type":23,"value":786},"检查脚本，检查超参是否有不合理的值",{"type":17,"tag":84,"props":788,"children":789},{},[790],{"type":23,"value":791},"检查模型结构是否实现正确",{"type":17,"tag":84,"props":793,"children":794},{},[795],{"type":23,"value":796},"检查输入数据是否正确",{"type":17,"tag":84,"props":798,"children":799},{},[800],{"type":23,"value":801},"检查loss曲线的收敛结果和收敛趋势是否存在异常",{"type":17,"tag":25,"props":803,"children":804},{},[805],{"type":23,"value":806},"若未发现问题，则应考虑优化超参取值。若多组超参均没有好的效果，则应考虑优化模型结构和算法，尝试新的idea。",{"type":17,"tag":60,"props":808,"children":810},{"id":809},"_2检查验证集准确率",[811],{"type":23,"value":812},"2、检查验证集准确率",{"type":17,"tag":25,"props":814,"children":815},{},[816],{"type":23,"value":817},"若训练集准确率和验证集准确率都未达到预期，则应首先参考上一节检查训练集准确率。若训练集准确率已达到预期，但是验证集准确率未达到预期，大概率是模型出现了过拟合，处理思路如下：",{"type":17,"tag":152,"props":819,"children":820},{},[821,826,831,836],{"type":17,"tag":84,"props":822,"children":823},{},[824],{"type":23,"value":825},"检查验证集评估脚本的评估逻辑有无错误。特别是数据处理方式是否与训练集一致，推理算法有误错误，是否加载了正确的模型checkpoint。",{"type":17,"tag":84,"props":827,"children":828},{},[829],{"type":23,"value":830},"增加数据量。包括增加样本量，进行数据增强和扰动等。",{"type":17,"tag":84,"props":832,"children":833},{},[834],{"type":23,"value":835},"正则化。常见的技术如参数范数惩罚（例如向目标函数中添加一个正则项），参数共享（模型的两个组件共享相同的参数值），提前中止训练等。",{"type":17,"tag":84,"props":837,"children":838},{},[839],{"type":23,"value":840},"适当降低模型的规模。例如减少卷积层数等。",{"type":17,"tag":60,"props":842,"children":844},{"id":843},"_3检查测试集准确率",[845],{"type":23,"value":846},"3、检查测试集准确率",{"type":17,"tag":25,"props":848,"children":849},{},[850],{"type":23,"value":851},"若验证集和测试集准确率都未达到预期，则应首先参考上一节检查验证集准确率。若验证集准确率已达到预期，但是测试集准确率未达到预期，考虑到测试集的数据是模型从未见过的新数据，原因一般是测试集的数据分布和训练集的数据分布不一致。处理思路如下：",{"type":17,"tag":152,"props":853,"children":854},{},[855,860],{"type":17,"tag":84,"props":856,"children":857},{},[858],{"type":23,"value":859},"检查测试集评估脚本的评估逻辑有误错误。特别是数据处理方式是否与训练集一致，推理算法有误错误，是否加载了正确的模型checkpoint。",{"type":17,"tag":84,"props":861,"children":862},{},[863],{"type":23,"value":864},"检查测试集中的数据质量，例如数据的分布范围是否明显同训练集不同，数据是否存在大量的噪声、缺失值或异常值。",{"type":17,"tag":60,"props":866,"children":868},{"id":867},"小结",[869],{"type":23,"value":867},{"type":17,"tag":25,"props":871,"children":872},{},[873],{"type":23,"value":874},"准确率低等问题的分析有章可循，首先检查代码和超参，然后检查模型结构，检查输入数据，检查loss曲线，最后确认准确率达到预期。由于准确率低可能存在多个原因，每一步中的问题修复后，都要重新执行训练和评估，确认是否还有其它的潜在问题。希望这些分析思路能起到良好的引导的作用，帮助你训练出满意的模型。",{"type":17,"tag":25,"props":876,"children":877},{},[878,880,886],{"type":23,"value":879},"注1：部分调试器检查功能仅在MindSpore调试器（",{"type":17,"tag":36,"props":881,"children":884},{"href":882,"rel":883},"https://mindspore.cn/tutorial/training/zh-CN/master/advanced_use/debugger.html",[40],[885],{"type":23,"value":882},{"type":23,"value":887},"）中可用。",{"type":17,"tag":25,"props":889,"children":890},{},[891,893,899],{"type":23,"value":892},"注2：文中截图全部来自MindSpore可视化调试调优工具MindInsight（",{"type":17,"tag":36,"props":894,"children":897},{"href":895,"rel":896},"https://www.mindspore.cn/mindinsight/docs/zh-CN/r1.8/index.html",[40],[898],{"type":23,"value":895},{"type":23,"value":900},"）。",{"title":7,"searchDepth":902,"depth":902,"links":903},4,[904,906,907,908,909,910,911,912,913,914,915],{"id":62,"depth":905,"text":62},2,{"id":258,"depth":905,"text":258},{"id":298,"depth":905,"text":298},{"id":334,"depth":905,"text":270},{"id":377,"depth":905,"text":275},{"id":443,"depth":905,"text":280},{"id":540,"depth":905,"text":285},{"id":763,"depth":905,"text":290},{"id":809,"depth":905,"text":812},{"id":843,"depth":905,"text":846},{"id":867,"depth":905,"text":867},"markdown","content:technology-blogs:zh:405.md","content","technology-blogs/zh/405.md","technology-blogs/zh/405","md",1776506137005]