[{"data":1,"prerenderedAt":861},["ShallowReactive",2],{"content-query-FkWMSU29eQ":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":855,"_id":856,"_source":857,"_file":858,"_stem":859,"_extension":860},"/technology-blogs/zh/394","zh",false,"","MindSpore模型精度调优应用（二）：精度调试调优思路","在模型的开发过程中，精度达不到预期常常让人头疼。为了帮助用户解决模型调试调优的问题，我们为MindSpore量身定做了可视化调试调优组件：MindInsight。我们还梳理了针对常见精度问题的调试调优指南，将以“MindSpore模型精度调优实战”系列文章的形式分享出来，希望能帮助用户轻松定位精度问题，快速优化模型精度。","2021-02-10","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/02/10/613c3922164049ad9377a7fae397f537.png","technology-blogs","调试调优",{"type":15,"children":16,"toc":846},"root",[17,25,31,36,41,46,51,56,61,66,71,76,82,89,94,99,105,110,119,124,132,137,142,148,153,158,163,168,173,178,185,190,196,201,206,211,216,221,226,231,236,244,249,254,269,274,280,285,290,295,300,305,310,315,320,325,330,335,340,345,353,358,366,371,376,381,387,392,397,402,407,412,417,422,427,432,437,442,447,452,457,468,476,481,486,494,499,504,512,517,529,534,540,545,595,601,606,647,653,658,664,669,677,682,687,721,729,734,742,747,766,772,777,782,787,796,802,813,824,835],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"mindspore模型精度调优应用二精度调试调优思路",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":28},"p",{},[29],{"type":24,"value":30},"MindSpore团队",{"type":18,"tag":26,"props":32,"children":33},{},[34],{"type":24,"value":35},"引子：在模型的开发过程中，精度达不到预期常常让人头疼。为了帮助用户解决模型调试调优的问题，我们为MindSpore量身定做了可视化调试调优组件：MindInsight。我们还梳理了针对常见精度问题的调试调优指南，将以“MindSpore模型精度调优应用”系列文章的形式分享出来，希望能帮助用户轻松定位精度问题，快速优化模型精度。",{"type":18,"tag":26,"props":37,"children":38},{},[39],{"type":24,"value":40},"本文是系列分享的第二篇，将给出常用的精度调试调优思路。本系列分享假设您的脚本已经能够运行并算出loss值。如果脚本还不能运行，请先参考相关报错提示进行修改。",{"type":18,"tag":26,"props":42,"children":43},{},[44],{"type":24,"value":45},"遇到精度问题时，常用调试调优思路如下：",{"type":18,"tag":26,"props":47,"children":48},{},[49],{"type":24,"value":50},"1. 检查代码和超参",{"type":18,"tag":26,"props":52,"children":53},{},[54],{"type":24,"value":55},"2. 检查模型结构",{"type":18,"tag":26,"props":57,"children":58},{},[59],{"type":24,"value":60},"3. 检查输入数据",{"type":18,"tag":26,"props":62,"children":63},{},[64],{"type":24,"value":65},"4. 检查loss曲线",{"type":18,"tag":26,"props":67,"children":68},{},[69],{"type":24,"value":70},"5. 检查精度是否达到预期",{"type":18,"tag":26,"props":72,"children":73},{},[74],{"type":24,"value":75},"代码是精度问题的重要源头，检查代码重在对脚本和代码做检查，力争在源头发现问题（第2节）；模型结构体现了MindSpore对代码的理解，检查模型结构重在检查MindSpore的理解和算法工程师的设计是否一致（第3节）；有的问题要到动态的训练过程中才会发现，检查输入数据（第4节）和loss曲线（第5节）正是将代码和动态训练现象结合进行检查；检查精度是否达到预期则是对整体精度调优过程重新审视，并考虑调整超参、解释模型、优化算法等调优手段（第6节）。此外，熟悉模型和工具也是很重要的（第1节）。下面将分别介绍这些思路。",{"type":18,"tag":19,"props":77,"children":79},{"id":78},"_0-精度调优准备",[80],{"type":24,"value":81},"0 精度调优准备",{"type":18,"tag":83,"props":84,"children":86},"h2",{"id":85},"_01-回顾算法设计全面熟悉模型",[87],{"type":24,"value":88},"0.1 回顾算法设计，全面熟悉模型",{"type":18,"tag":26,"props":90,"children":91},{},[92],{"type":24,"value":93},"精度调优前，要先对算法设计做回顾，确保算法设计明确。如果参考论文实现模型，则应回顾论文中的全部设计细节和超参选择情况；如果参考其它框架脚本实现模型，则应确保有一个唯一的、精度能够达标的标杆脚本；如果是新开发的算法，也应将重要的设计细节和超参选择明确出来。这些信息是后面检查脚本步骤的重要依据。",{"type":18,"tag":26,"props":95,"children":96},{},[97],{"type":24,"value":98},"精度调优前，还要全面熟悉模型。只有熟悉了模型，才能准确理解MindInsight提供的信息，判断是否存在问题，查找问题源头。因此，花时间理解模型算法和结构、理解模型中算子的作用和参数的含义、理解模型所用优化器的特性等模型要素是很重要的。动手分析精度问题细节前，建议先带着问题加深对这些模型要素的了解。",{"type":18,"tag":83,"props":100,"children":102},{"id":101},"_02-熟悉工具",[103],{"type":24,"value":104},"0.2 熟悉工具",{"type":18,"tag":26,"props":106,"children":107},{},[108],{"type":24,"value":109},"MindInsight功能丰富，建议用户简单阅读MindInsight教程，了解主要功能。定位精度问题时，建议使能summary训练信息收集功能，在脚本中加入SummaryCollector，并使用训练看板查看训练过程数据，如下图所示。",{"type":18,"tag":26,"props":111,"children":112},{},[113],{"type":18,"tag":114,"props":115,"children":118},"img",{"alt":116,"src":117},"image1.png","https://bbs-img-cbc-cn.obs.cn-north-1.myhuaweicloud.com/data/attachment/forum/202102/09/2121363hgfxjinyo9sj5jd.png",[],{"type":18,"tag":26,"props":120,"children":121},{},[122],{"type":24,"value":123},"图 1在训练脚本中加入SummaryCollector以使用MindInsight",{"type":18,"tag":26,"props":125,"children":126},{},[127],{"type":18,"tag":114,"props":128,"children":131},{"alt":129,"src":130},"image3.png","https://bbs-img.huaweicloud.com/data/forums/attachment/forum/202103/25/2136505wutrkgjwtp6nkvi.png",[],{"type":18,"tag":26,"props":133,"children":134},{},[135],{"type":24,"value":136},"图 2 训练看板",{"type":18,"tag":26,"props":138,"children":139},{},[140],{"type":24,"value":141},"当您需要在线调试模型时，请参考官网使能调试器功能。",{"type":18,"tag":19,"props":143,"children":145},{"id":144},"_1-检查代码和超参",[146],{"type":24,"value":147},"1 检查代码和超参",{"type":18,"tag":26,"props":149,"children":150},{},[151],{"type":24,"value":152},"代码是精度问题的重要源头，超参问题、模型结构问题、数据问题、算法设计和实现问题会体现在脚本中，对脚本做检查是定位精度问题很有效率的手段。检查代码主要依赖代码走读，建议使用小黄鸭调试法：在代码走读的过程中，耐心地向没有经验的“小黄鸭”解释每一行代码的作用，从而激发灵感，发现代码问题。检查脚本时，要注意检查脚本实现（包括数据处理、模型结构、loss函数、优化器等实现）同设计是否一致，如果参考了其它脚本，要重点检查脚本实现同其它脚本是否一致，所有不一致的地方都应该有充分合理的理由，否则就应修改。",{"type":18,"tag":26,"props":154,"children":155},{},[156],{"type":24,"value":157},"检查脚本时，也要关注超参的情况，超参问题主要体现为超参取值不合理，例如",{"type":18,"tag":26,"props":159,"children":160},{},[161],{"type":24,"value":162},"学习率设置不合理；",{"type":18,"tag":26,"props":164,"children":165},{},[166],{"type":24,"value":167},"loss_scale参数不合理；",{"type":18,"tag":26,"props":169,"children":170},{},[171],{"type":24,"value":172},"权重初始化参数不合理等。",{"type":18,"tag":26,"props":174,"children":175},{},[176],{"type":24,"value":177},"MindInsight可以辅助用户对超参做检查，大多数情况下，SummaryCollector会自动记录常见超参，您可以通过MindInsight的训练参数详情功能（如下图）和溯源分析功能查看超参。结合MindInsight模型溯源分析模块和脚本中的代码，可以确认超参的取值，识别明显不合理的超参。如果有标杆脚本，建议同标杆脚本一一比对超参取值，如果有默认参数值，则默认值也应一并比对，以避免不同框架的参数默认值不同导致精度下降或者训练错误。",{"type":18,"tag":26,"props":179,"children":180},{},[181],{"type":18,"tag":114,"props":182,"children":184},{"alt":129,"src":183},"https://bbs-img.huaweicloud.com/data/forums/attachment/forum/202103/25/213726bccu40q5zb57jgh6.png",[],{"type":18,"tag":26,"props":186,"children":187},{},[188],{"type":24,"value":189},"图 3通过MindInsight训练参数详情查看模型超参",{"type":18,"tag":19,"props":191,"children":193},{"id":192},"_2-检查模型结构",[194],{"type":24,"value":195},"2 检查模型结构",{"type":18,"tag":26,"props":197,"children":198},{},[199],{"type":24,"value":200},"在模型结构方面，常见的问题有：",{"type":18,"tag":26,"props":202,"children":203},{},[204],{"type":24,"value":205},"算子使用错误（使用的算子不适用于目标场景，如应该使用浮点除，错误地使用了整数除）；",{"type":18,"tag":26,"props":207,"children":208},{},[209],{"type":24,"value":210},"权重共享错误（共享了不应共享的权重）；",{"type":18,"tag":26,"props":212,"children":213},{},[214],{"type":24,"value":215},"权重冻结错误（冻结了不应冻结的权重）；",{"type":18,"tag":26,"props":217,"children":218},{},[219],{"type":24,"value":220},"节点连接错误（应该连接到计算图中的block未连接）；",{"type":18,"tag":26,"props":222,"children":223},{},[224],{"type":24,"value":225},"loss函数错误；",{"type":18,"tag":26,"props":227,"children":228},{},[229],{"type":24,"value":230},"优化器算法错误（如果自行实现了优化器）等。",{"type":18,"tag":26,"props":232,"children":233},{},[234],{"type":24,"value":235},"建议通过检查模型代码的方式对模型结构进行检查。此外，MindInsight也可以辅助用户对模型结构进行检查。大多数情况下，SummaryCollector会自动记录计算图，通过MindInsight，用户可以方便地对计算图进行查看。模型脚本运行后，建议使用MindInsight计算图可视模块查看模型结构，加深对计算图的理解，确认模型结构符合预期。若有标杆脚本，还可以同标杆脚本对照查看计算图，检查当前脚本和标杆脚本的计算图是否存在重要的差异。",{"type":18,"tag":26,"props":237,"children":238},{},[239],{"type":18,"tag":114,"props":240,"children":243},{"alt":241,"src":242},"image4.png","https://bbs-img-cbc-cn.obs.cn-north-1.myhuaweicloud.com/data/attachment/forum/202102/09/211145voegsvkjvcwbdpvf.png",[],{"type":18,"tag":26,"props":245,"children":246},{},[247],{"type":24,"value":248},"图 4 通过MindInsight训练看板中的计算图模块查看模型结构",{"type":18,"tag":26,"props":250,"children":251},{},[252],{"type":24,"value":253},"考虑到模型结构一般都很复杂，期望在这一步就能发现所有的模型结构问题是不现实的。只要通过可视化的模型结构加深对计算图的理解，发现明显的结构问题即可。后面的步骤中，发现了更明确的精度问题现象后，我们还会回到这一步重新检查确认。",{"type":18,"tag":26,"props":255,"children":256},{},[257,259,267],{"type":24,"value":258},"注1：MindInsight支持查看SummaryCollector记录的计算图和MindSpore context的save_graphs参数导出的pb文件计算图。请参考我们教程中的“计算图可视化”部分了解更多信息（",{"type":18,"tag":260,"props":261,"children":265},"a",{"href":262,"rel":263},"https://www.mindspore.cn/tutorial/training/zh-CN/master/advanced_use/dashboard.html",[264],"nofollow",[266],{"type":24,"value":262},{"type":24,"value":268},"）。",{"type":18,"tag":26,"props":270,"children":271},{},[272],{"type":24,"value":273},"注2：脚本迁移工具可以将PyTorch、TensorFlow框架下编写的模型转换为MindSpore脚本。",{"type":18,"tag":19,"props":275,"children":277},{"id":276},"_3-检查输入数据",[278],{"type":24,"value":279},"3 检查输入数据",{"type":18,"tag":26,"props":281,"children":282},{},[283],{"type":24,"value":284},"通过检查输入模型的数据，可以结合脚本判断数据处理流水线和数据集是否存在问题。输入数据的常见问题有：",{"type":18,"tag":26,"props":286,"children":287},{},[288],{"type":24,"value":289},"数据缺失值过多；",{"type":18,"tag":26,"props":291,"children":292},{},[293],{"type":24,"value":294},"每个类别中的样本数目不均衡；",{"type":18,"tag":26,"props":296,"children":297},{},[298],{"type":24,"value":299},"数据中存在异常值；",{"type":18,"tag":26,"props":301,"children":302},{},[303],{"type":24,"value":304},"数据标签错误；",{"type":18,"tag":26,"props":306,"children":307},{},[308],{"type":24,"value":309},"训练样本不足；",{"type":18,"tag":26,"props":311,"children":312},{},[313],{"type":24,"value":314},"未对数据进行标准化，输入模型的数据不在正确的范围内；",{"type":18,"tag":26,"props":316,"children":317},{},[318],{"type":24,"value":319},"finetune和pretrain的数据处理方式不同；",{"type":18,"tag":26,"props":321,"children":322},{},[323],{"type":24,"value":324},"训练阶段和推理阶段的数据处理方式不同；",{"type":18,"tag":26,"props":326,"children":327},{},[328],{"type":24,"value":329},"数据处理参数不正确等。",{"type":18,"tag":26,"props":331,"children":332},{},[333],{"type":24,"value":334},"MindInsight可以辅助用户对输入数据、数据处理流水线进行检查。大多数情况下，SummaryCollector会自动记录输入模型的数据（数据处理后的数据）和数据处理流水线参数。输入模型的数据会展示在“数据抽样”模块，数据处理流水线参数会展示在“数据图”模块和“数据溯源”模块。",{"type":18,"tag":26,"props":336,"children":337},{},[338],{"type":24,"value":339},"通过MindInsight的数据抽样模块，可以检查输入模型的（数据处理流水线处理后的）数据。若数据明显不符合预期（例如数据被裁剪的范围过大，数据旋转的角度过大等），可以判断输入数据出现了一定的问题。",{"type":18,"tag":26,"props":341,"children":342},{},[343],{"type":24,"value":344},"通过MindInsight的数据图和数据溯源模块，可以检查数据处理流水线的数据处理过程和具体参数取值，从而发现不合理的数据处理方法。",{"type":18,"tag":26,"props":346,"children":347},{},[348],{"type":18,"tag":114,"props":349,"children":352},{"alt":350,"src":351},"image5.png","https://bbs-img-cbc-cn.obs.cn-north-1.myhuaweicloud.com/data/attachment/forum/202102/09/211153rzyx3kmnagbkox7d.png",[],{"type":18,"tag":26,"props":354,"children":355},{},[356],{"type":24,"value":357},"图 5通过MindInsight训练看板中的数据图查看数据处理流水线",{"type":18,"tag":26,"props":359,"children":360},{},[361],{"type":18,"tag":114,"props":362,"children":365},{"alt":363,"src":364},"image6.png","https://bbs-img.huaweicloud.com/data/forums/attachment/forum/202104/26/190310lababffc1gyr78lt.png",[],{"type":18,"tag":26,"props":367,"children":368},{},[369],{"type":24,"value":370},"图 6通过MindInsight训练看板中的数据图查看数据处理流水线",{"type":18,"tag":26,"props":372,"children":373},{},[374],{"type":24,"value":375},"如果有标杆脚本，还可以同标杆脚本对照，检查数据处理流水线输出的数据是否和当前脚本的数据相同。例如，将数据处理流水线输出的数据保存为npy文件，然后使用numpy.allclose()方法对标杆脚本和当前脚本的数据进行对比。如果发现不同，则数据处理阶段可能存在精度问题。",{"type":18,"tag":26,"props":377,"children":378},{},[379],{"type":24,"value":380},"若数据处理流水线未发现问题，可以手动检查数据集是否存在分类不均衡、标签匹配错误、缺失值过多、训练样本不足等问题。",{"type":18,"tag":19,"props":382,"children":384},{"id":383},"_4-检查loss曲线",[385],{"type":24,"value":386},"4 检查loss曲线",{"type":18,"tag":26,"props":388,"children":389},{},[390],{"type":24,"value":391},"很多精度问题会在网络训练过程中发现，常见的问题或现象有：",{"type":18,"tag":26,"props":393,"children":394},{},[395],{"type":24,"value":396},"权重初始化不合理（例如初始值为0，初始值范围不合理等）；",{"type":18,"tag":26,"props":398,"children":399},{},[400],{"type":24,"value":401},"权重中存在过大、过小值；",{"type":18,"tag":26,"props":403,"children":404},{},[405],{"type":24,"value":406},"权重变化过大；",{"type":18,"tag":26,"props":408,"children":409},{},[410],{"type":24,"value":411},"权重冻结不正确；",{"type":18,"tag":26,"props":413,"children":414},{},[415],{"type":24,"value":416},"权重共享不正确；",{"type":18,"tag":26,"props":418,"children":419},{},[420],{"type":24,"value":421},"激活值饱和或过弱（例如Sigmoid的输出接近1，Relu的输出全为0）；",{"type":18,"tag":26,"props":423,"children":424},{},[425],{"type":24,"value":426},"梯度爆炸、消失；",{"type":18,"tag":26,"props":428,"children":429},{},[430],{"type":24,"value":431},"训练epoch不足；",{"type":18,"tag":26,"props":433,"children":434},{},[435],{"type":24,"value":436},"算子计算结果存在NAN、INF；",{"type":18,"tag":26,"props":438,"children":439},{},[440],{"type":24,"value":441},"算子计算过程溢出（计算过程中的溢出不一定都是有害的）等。",{"type":18,"tag":26,"props":443,"children":444},{},[445],{"type":24,"value":446},"上述这些问题或现象，有的可以通过loss表现出来，有的则难以观察。MindInsight提供了针对性的功能，可以观察上述现象、自动检查问题，帮助您更快定位问题根因。例如：",{"type":18,"tag":26,"props":448,"children":449},{},[450],{"type":24,"value":451},"l MindInsight的参数分布图模块可以展示模型权重随训练过程的变化趋势；",{"type":18,"tag":26,"props":453,"children":454},{},[455],{"type":24,"value":456},"l MindInsight的张量可视模块可以展示张量的具体取值，对不同张量进行对比；",{"type":18,"tag":26,"props":458,"children":459},{},[460,462],{"type":24,"value":461},"l MindInsight调试器内置了种类丰富，功能强大的检查能力，可以检查权重问题（例如权重不更新、权重更新过大、权重值过大/过小）、梯度问题（例如梯度消失、梯度爆炸）、激活值问题（例如激活值饱和或过弱）、张量全为0、NAN/INF、算子计算过程溢出等问题。调试器使用教程： ",{"type":18,"tag":260,"props":463,"children":466},{"href":464,"rel":465},"https://www.mindspore.cn/tutorial/training/zh-CN/master/advanced_use/debugger.html",[264],[467],{"type":24,"value":464},{"type":18,"tag":26,"props":469,"children":470},{},[471],{"type":18,"tag":114,"props":472,"children":475},{"alt":473,"src":474},"image7.png","https://bbs-img-cbc-cn.obs.cn-north-1.myhuaweicloud.com/data/attachment/forum/202102/09/211207mffokqtnbfmvxe3a.png",[],{"type":18,"tag":26,"props":477,"children":478},{},[479],{"type":24,"value":480},"图 7通过MindInsight训练看板中的标量可视模块查看loss曲线",{"type":18,"tag":26,"props":482,"children":483},{},[484],{"type":24,"value":485},"大多数情况下，SummaryCollector会自动记录模型的loss曲线，可以通过MindInsight的标量可视模块查看。loss曲线能够反映网络训练的动态趋势，通过观察loss曲线，可以得到模型是否收敛、是否过拟合等信息。",{"type":18,"tag":26,"props":487,"children":488},{},[489],{"type":18,"tag":114,"props":490,"children":493},{"alt":491,"src":492},"image8.png","https://bbs-img-cbc-cn.obs.cn-north-1.myhuaweicloud.com/data/attachment/forum/202102/09/211214zpfgnyuydhkvypbi.png",[],{"type":18,"tag":26,"props":495,"children":496},{},[497],{"type":24,"value":498},"图 8通过MindInsight参数分布图可以查看训练过程中的权重变化情况",{"type":18,"tag":26,"props":500,"children":501},{},[502],{"type":24,"value":503},"大多数情况下，SummaryCollector会自动记录模型参数变化情况（默认记录5个参数），可以通过MindInsight的参数分布图模块查看。如果想要记录更多参数的参数分布图，请参考SummaryCollector的histogram_regular参数，或参考HistogramSummary算子。",{"type":18,"tag":26,"props":505,"children":506},{},[507],{"type":18,"tag":114,"props":508,"children":511},{"alt":509,"src":510},"image9.png","https://bbs-img-cbc-cn.obs.cn-north-1.myhuaweicloud.com/data/attachment/forum/202102/09/211220ijgd4m7azgaczhxn.png",[],{"type":18,"tag":26,"props":513,"children":514},{},[515],{"type":24,"value":516},"图 9通过MindInsight训练看板中的张量可视模块查看特定张量的具体取值",{"type":18,"tag":26,"props":518,"children":519},{},[520,522,528],{"type":24,"value":521},"张量不会被自动记录，如果想要通过MindInsight查看张量的具体取值，请使用TensorSummary算子 （",{"type":18,"tag":260,"props":523,"children":526},{"href":524,"rel":525},"https://www.mindspore.cn/tutorial/training/zh-CN/master/advanced_use/summary_record.html#summarysummarycollector",[264],[527],{"type":24,"value":524},{"type":24,"value":268},{"type":18,"tag":26,"props":530,"children":531},{},[532],{"type":24,"value":533},"下面结合loss曲线的常见现象介绍使用MindInsight进行精度问题定位的思路。",{"type":18,"tag":83,"props":535,"children":537},{"id":536},"_41-loss跑飞",[538],{"type":24,"value":539},"4.1 loss跑飞",{"type":18,"tag":26,"props":541,"children":542},{},[543],{"type":24,"value":544},"loss跑飞是指loss中出现了NAN、+/-INF或者特别大的值。loss跑飞一般意味着算法设计或实现存在问题。定位思路如下：",{"type":18,"tag":546,"props":547,"children":548},"ol",{},[549,555,560,565,570,575,580,585,590],{"type":18,"tag":550,"props":551,"children":552},"li",{},[553],{"type":24,"value":554},"回顾脚本、模型结构和数据，",{"type":18,"tag":550,"props":556,"children":557},{},[558],{"type":24,"value":559},"检查超参是否有不合理的特别大/特别小的取值，",{"type":18,"tag":550,"props":561,"children":562},{},[563],{"type":24,"value":564},"检查模型结构是否实现正确，特别是检查loss函数是否实现正确，",{"type":18,"tag":550,"props":566,"children":567},{},[568],{"type":24,"value":569},"检查输入数据中是否有缺失值、是否有特别大/特别小的取值。",{"type":18,"tag":550,"props":571,"children":572},{},[573],{"type":24,"value":574},"观察训练看板中的参数分布图，检查参数更新是否有明显的异常。若发现参数更新异常，可以结合调试器定位参数更新异常的原因。",{"type":18,"tag":550,"props":576,"children":577},{},[578],{"type":24,"value":579},"使用调试器模块对训练现场进行检查。",{"type":18,"tag":550,"props":581,"children":582},{},[583],{"type":24,"value":584},"若loss值出现NAN、+/-INF，可使用“检查张量溢出”条件添加全局监测点，定位首先出现NAN、+/-INF的算子节点，检查算子的输入数据是否会导致计算异常（例如除零）。若是算子输入数据的问题，则可以针对性地加入小数值epsilon避免计算异常。",{"type":18,"tag":550,"props":586,"children":587},{},[588],{"type":24,"value":589},"若loss值出现特别大的值，可使用“检查过大张量”条件添加全局监测点，定位首先出现大值的算子节点，检查算子的输入数据是否会导致计算异常。若输入数据本身存在异常，则可以继续向上查询产生该输入数据的算子，直到定位出具体原因。",{"type":18,"tag":550,"props":591,"children":592},{},[593],{"type":24,"value":594},"若怀疑参数更新、梯度等方面存在异常，可使用“检查权重变化过大”、“检查梯度消失”、“检查梯度过大”等条件设置监测点，定位到异常的权重或梯度，然后结合张量检查视图，逐层向上对可疑的正向算子、反向算子、优化器算子等进行检查。",{"type":18,"tag":83,"props":596,"children":598},{"id":597},"_42-loss收敛慢",[599],{"type":24,"value":600},"4.2 loss收敛慢",{"type":18,"tag":26,"props":602,"children":603},{},[604],{"type":24,"value":605},"loss收敛慢是指loss震荡、收敛速度慢，经过很长时间才能达到预期值，或者最终也无法收敛到预期值。相较于loss跑飞，loss收敛慢的数值特征不明显，更难定位。定位思路如下：",{"type":18,"tag":546,"props":607,"children":608},{},[609,613,618,623,628,632,637,642],{"type":18,"tag":550,"props":610,"children":611},{},[612],{"type":24,"value":554},{"type":18,"tag":550,"props":614,"children":615},{},[616],{"type":24,"value":617},"检查超参是否有不合理的特别大/特别小的取值，特别是检查学习率是否设置过小或过大，学习率设置过小会导致收敛速度慢，学习率设置过大会导致loss震荡、不下降；",{"type":18,"tag":550,"props":619,"children":620},{},[621],{"type":24,"value":622},"检查模型结构是否实现正确，特别是检查loss函数、优化器是否实现正确；",{"type":18,"tag":550,"props":624,"children":625},{},[626],{"type":24,"value":627},"检查输入数据的范围是否正常，特别是输入数据的值是否过小",{"type":18,"tag":550,"props":629,"children":630},{},[631],{"type":24,"value":574},{"type":18,"tag":550,"props":633,"children":634},{},[635],{"type":24,"value":636},"使用调试器模块对训练现场进程检查。",{"type":18,"tag":550,"props":638,"children":639},{},[640],{"type":24,"value":641},"可使用“检查权重变化过小”、“检查未变化权重”条件对可训练（未固定）的权重进行监测，检查权重是否变化过小。若发现权重变化过小，可进一步检查学习率取值是否过小、优化器算法是否正确实现、梯度是否消失，并做针对性的修复。",{"type":18,"tag":550,"props":643,"children":644},{},[645],{"type":24,"value":646},"可使用“检查梯度消失”条件对梯度进行监测，检查是否存在梯度消失的现象。若发现梯度消失，可进一步向上检查导致梯度消失的原因。例如，可以通过“检查激活值范围”条件检查是否出现了激活值饱和、Relu输出为0等问题。",{"type":18,"tag":83,"props":648,"children":650},{"id":649},"_43-其它loss现象",[651],{"type":24,"value":652},"4.3 其它loss现象",{"type":18,"tag":26,"props":654,"children":655},{},[656],{"type":24,"value":657},"若训练集上loss为0，一般说明模型出现了过拟合，请尝试增大训练集大小。",{"type":18,"tag":19,"props":659,"children":661},{"id":660},"_5-检查精度是否达到预期",[662],{"type":24,"value":663},"5 检查精度是否达到预期",{"type":18,"tag":26,"props":665,"children":666},{},[667],{"type":24,"value":668},"MindInsight可以为用户记录每次训练的精度结果。在model.train和model.eval中使用同一个SummaryCollector实例时，会自动记录模型评估（metrics）信息。训练结束后，可以通过MindInsight的模型溯源模块检查训练结果精度是否达标。",{"type":18,"tag":26,"props":670,"children":671},{},[672],{"type":18,"tag":114,"props":673,"children":676},{"alt":674,"src":675},"image10.png","https://bbs-img-cbc-cn.obs.cn-north-1.myhuaweicloud.com/data/attachment/forum/202102/09/211309ndbvavlgos82ol8r.png",[],{"type":18,"tag":26,"props":678,"children":679},{},[680],{"type":24,"value":681},"图 10通过MindInsight溯源分析功能查看模型评估信息",{"type":18,"tag":26,"props":683,"children":684},{},[685],{"type":24,"value":686},"若模型的loss值、metric值未达到预期，可以参考以下思路进行定位和优化：",{"type":18,"tag":688,"props":689,"children":690},"ul",{},[691,696,701,706,711,716],{"type":18,"tag":550,"props":692,"children":693},{},[694],{"type":24,"value":695},"回顾代码、模型结构、输入数据和loss曲线，",{"type":18,"tag":550,"props":697,"children":698},{},[699],{"type":24,"value":700},"检查超参是否有不合理的值",{"type":18,"tag":550,"props":702,"children":703},{},[704],{"type":24,"value":705},"检查模型结构是否实现正确",{"type":18,"tag":550,"props":707,"children":708},{},[709],{"type":24,"value":710},"检查输入数据是否正确",{"type":18,"tag":550,"props":712,"children":713},{},[714],{"type":24,"value":715},"检查loss曲线的收敛结果和收敛趋势是否存在异常",{"type":18,"tag":550,"props":717,"children":718},{},[719],{"type":24,"value":720},"尝试使用MindInsight溯源分析功能优化超参。溯源分析页面会对超参的重要性进行分析，用户应优先考虑调整重要性高的超参，从散点图中可以观察出超参和优化目标的关系，从而针对性地调整超参取值。",{"type":18,"tag":26,"props":722,"children":723},{},[724],{"type":18,"tag":114,"props":725,"children":728},{"alt":726,"src":727},"image11.png","https://bbs-img-cbc-cn.obs.cn-north-1.myhuaweicloud.com/data/attachment/forum/202102/09/211314i4d7tlina9kg5cq4.png",[],{"type":18,"tag":26,"props":730,"children":731},{},[732],{"type":24,"value":733},"图 11通过MindInsight溯源分析查看参数重要性",{"type":18,"tag":26,"props":735,"children":736},{},[737],{"type":18,"tag":114,"props":738,"children":741},{"alt":739,"src":740},"image12.png","https://bbs-img-cbc-cn.obs.cn-north-1.myhuaweicloud.com/data/attachment/forum/202102/09/211318leub5zifzcodicdr.png",[],{"type":18,"tag":26,"props":743,"children":744},{},[745],{"type":24,"value":746},"图 12通过MindInsight溯源分析以散点图形式查看参数和优化目标的关系",{"type":18,"tag":688,"props":748,"children":749},{},[750,761],{"type":18,"tag":550,"props":751,"children":752},{},[753,755],{"type":24,"value":754},"尝试使用MindInsight调参器优化超参。请注意，调参器通过执行多次完整训练的方式进行超参搜索，消耗的时间为网络一次训练用时的若干倍，如果网络一次训练耗时较长，则超参搜索将需要很长的时间。调参器使用教程： ",{"type":18,"tag":260,"props":756,"children":759},{"href":757,"rel":758},"https://www.mindspore.cn/tutorial/training/zh-CN/master/advanced_use/hyper_parameters_auto_tuning.html",[264],[760],{"type":24,"value":757},{"type":18,"tag":550,"props":762,"children":763},{},[764],{"type":24,"value":765},"尝试使用MindInsight模型解释功能优化模型数据集。模型解释功能可以通过显著图可视化展示对分类结果最重要的区域，还可以通过评分体系提示应该对哪类标签进行优化。",{"type":18,"tag":19,"props":767,"children":769},{"id":768},"_6-小结",[770],{"type":24,"value":771},"6 小结",{"type":18,"tag":26,"props":773,"children":774},{},[775],{"type":24,"value":776},"由于相同的现象存在多个可能原因，精度问题的定位非常依赖专家经验。希望上述定位方法和功能能够起到良好的引导的作用，帮助你不断积累成功经验，成为精度调优大师。",{"type":18,"tag":26,"props":778,"children":779},{},[780],{"type":24,"value":781},"7 往期回顾",{"type":18,"tag":26,"props":783,"children":784},{},[785],{"type":24,"value":786},"MindSpore模型精度调优应用（一）：精度问题的常见现象、原因和简要调优思路",{"type":18,"tag":26,"props":788,"children":789},{},[790],{"type":18,"tag":260,"props":791,"children":794},{"href":792,"rel":793},"https://www.mindspore.cn/news/newschildren?id=381",[264],[795],{"type":24,"value":792},{"type":18,"tag":19,"props":797,"children":799},{"id":798},"_8-欢迎关注我们",[800],{"type":24,"value":801},"8 欢迎关注我们",{"type":18,"tag":26,"props":803,"children":804},{},[805,807],{"type":24,"value":806},"MindSpore官网： ",{"type":18,"tag":260,"props":808,"children":811},{"href":809,"rel":810},"https://www.mindspore.cn/",[264],[812],{"type":24,"value":809},{"type":18,"tag":26,"props":814,"children":815},{},[816,818],{"type":24,"value":817},"MindSpore代码仓库： ",{"type":18,"tag":260,"props":819,"children":822},{"href":820,"rel":821},"https://gitee.com/mindspore/mindspore",[264],[823],{"type":24,"value":820},{"type":18,"tag":26,"props":825,"children":826},{},[827,829],{"type":24,"value":828},"MindInsight代码仓库： ",{"type":18,"tag":260,"props":830,"children":833},{"href":831,"rel":832},"https://gitee.com/mindspore/mindinsight",[264],[834],{"type":24,"value":831},{"type":18,"tag":26,"props":836,"children":837},{},[838,840],{"type":24,"value":839},"MindInsight使用教程：",{"type":18,"tag":260,"props":841,"children":844},{"href":842,"rel":843},"https://www.mindspore.cn/mindinsight/docs/zh-CN/r1.8/index.html",[264],[845],{"type":24,"value":842},{"title":7,"searchDepth":847,"depth":847,"links":848},4,[849,851,852,853,854],{"id":85,"depth":850,"text":88},2,{"id":101,"depth":850,"text":104},{"id":536,"depth":850,"text":539},{"id":597,"depth":850,"text":600},{"id":649,"depth":850,"text":652},"markdown","content:technology-blogs:zh:394.md","content","technology-blogs/zh/394.md","technology-blogs/zh/394","md",1776506136790]