[{"data":1,"prerenderedAt":155},["ShallowReactive",2],{"content-query-r4Gy6Yfn1v":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":149,"_id":150,"_source":151,"_file":152,"_stem":153,"_extension":154},"/technology-blogs/zh/1866","zh",false,"","【MindSpore易点通】调优精度之低阶API使用静态LossScale","在混合精度中，使用float16类型来替代float32类型存储数据，从而达到减少内存和提高计算速度的效果","2022-09-22","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2022/09/30/575058a12ce049989010cd194ff67e4b.png","technology-blogs","调试调优",{"type":15,"children":16,"toc":142},"root",[17,25,31,37,53,59,64,74,80,85,90,95,100,105,110,115,120,125,130],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"mindspore易点通调优精度之低阶api使用静态lossscale",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":29},"h3",{"id":28},"背景信息",[30],{"type":24,"value":28},{"type":18,"tag":32,"props":33,"children":34},"p",{},[35],{"type":24,"value":36},"在混合精度中，使用float16类型来替代float32类型存储数据，从而达到减少内存和提高计算速度的效果。但是由于float16类型要比float32类型表示的范围小很多，所以当某些参数（比如说梯度）在训练过程中变得很小时，就会发生数据下溢的情况。LossScale的主要思想是在计算loss时，将loss扩大一定的倍数，由于链式法则的存在，梯度也会相应扩大，然后在优化器更新权重时再缩小相应的倍数，从而避免了数据下溢的情况又不影响计算结果；而使用静态LossScale则是静态缩放更新网络，使用固定的loss scale值；在低阶模型中实现静态loss scale功能时，通常结合TrainOneStepWithLossScaleCell来实现。",{"type":18,"tag":32,"props":38,"children":39},{},[40,42,51],{"type":24,"value":41},"在MindSpore中，loss scale的使用方法又分动态loss scale和静态loss scale两种，二者具体区别详见",{"type":18,"tag":43,"props":44,"children":48},"a",{"href":45,"rel":46},"https://6a3d5ce1-15b9-41a1-917f-4b8faf9770dc.vscode-webview-test.com/vscode-resourcehttps://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file///c:/Users/hangtian/Downloads/MindBook%E7%AD%9B%E9%80%890803/MindBook_to_MS_tuomin/%E5%BC%80%E5%8F%91%E6%8C%87%E5%AF%BC%E6%89%8B%E5%86%8C/%E7%B2%BE%E5%BA%A6%E8%B0%83%E4%BC%98/%E8%AE%BE%E7%BD%AELossScale%E8%B0%83%E4%BC%98%E7%B2%BE%E5%BA%A6/%E9%9D%99%E6%80%81LossScale%E5%92%8C%E5%8A%A8%E6%80%81LossScale%E7%9A%84%E5%8C%BA%E5%88%AB/%E9%9D%99%E6%80%81LossScale%E5%92%8C%E5%8A%A8%E6%80%81LossScale%E7%9A%84%E5%8C%BA%E5%88%AB.md",[47],"nofollow",[49],{"type":24,"value":50},"静态LossScale和动态LossScale的区别",{"type":24,"value":52},"。",{"type":18,"tag":26,"props":54,"children":56},{"id":55},"_1示例代码段",[57],{"type":24,"value":58},"1、示例代码段",{"type":18,"tag":32,"props":60,"children":61},{},[62],{"type":24,"value":63},"**步骤：**在使用静态LossScale功能前，首先要定义一张网络、损失函数、优化器，然后将损失函数融入到神经网络中，再使用静态LossScale功能",{"type":18,"tag":65,"props":66,"children":68},"pre",{"code":67},"from src.new import TrainOneStepWithLossScaleCell\n\n# Define the loss function\n\n    loss_function = nn.loss.SoftmaxCrossEntropyWithLogits(sparse=True,\n\n                                                reduction='mean')\n\n\n\n    # Define the optimizer\n\n    opt = nn.SGD(net.trainable_params(), LR_ORI, MOMENTUM_ORI, WEIGHT_DECAY)\n\n\n\n# Bind loss_function to net\n\n    model_constructed = BuildTrainNetwork(net, loss_function,\n\n                        TRAIN_BATCH_SIZE, CLASS_NUM)\n\n    \n\n    # Define the Dynamic Loss scale update cell\n\n    loss_scale_manager = nn.FixedLossScaleUpdateCell(loss_scale_value=2**12)\n\n    # Define Network training with loss scaling\n\n    model_constructed = TrainOneStepWithLossScaleCell(network=model_constructed,      optimizer=opt, scale_sense=loss_scale_manager)\n\n\n\n    # Train\n\n    train_net(model_constructed, net, loss_function, EPOCH_MAX, TRAIN_PATH,\n\n        VAL_PATH, TRAIN_BATCH_SIZE, VAL_BATCH_SIZE, REPEAT_SIZE)\n",[69],{"type":18,"tag":70,"props":71,"children":72},"code",{"__ignoreMap":7},[73],{"type":24,"value":67},{"type":18,"tag":26,"props":75,"children":77},{"id":76},"_2代码解析",[78],{"type":24,"value":79},"2、代码解析",{"type":18,"tag":32,"props":81,"children":82},{},[83],{"type":24,"value":84},"loss_scale_manager = nn.FixedLossScaleUpdateCell(loss_scale_value=2**12)",{"type":18,"tag":32,"props":86,"children":87},{},[88],{"type":24,"value":89},"接口作用：静态scale更新网络，loss scaling 值将不会被更新。",{"type":18,"tag":32,"props":91,"children":92},{},[93],{"type":24,"value":94},"参数详解：",{"type":18,"tag":32,"props":96,"children":97},{},[98],{"type":24,"value":99},"loss_scale_value :表示初始化的LossScale，float数据类型；",{"type":18,"tag":32,"props":101,"children":102},{},[103],{"type":24,"value":104},"model_constructed = TrainOneStepWithLossScaleCell(network=model_constructed, optimizer=opt, scale_sense=loss_scale_manager)",{"type":18,"tag":32,"props":106,"children":107},{},[108],{"type":24,"value":109},"功能：：定义一个静态LossScale的模型控制器，它将网络、优化器和可能的scale 更新网络作为参数。",{"type":18,"tag":32,"props":111,"children":112},{},[113],{"type":24,"value":114},"接口参数详解：",{"type":18,"tag":32,"props":116,"children":117},{},[118],{"type":24,"value":119},"network:为训练网络，只支持单一输出；",{"type":18,"tag":32,"props":121,"children":122},{},[123],{"type":24,"value":124},"optimizer:为用户更新权重的优化器；",{"type":18,"tag":32,"props":126,"children":127},{},[128],{"type":24,"value":129},"scale_sense:为更新逻辑单元。其可为Cell类型或Tensor类型；",{"type":18,"tag":32,"props":131,"children":132},{},[133,135,141],{"type":24,"value":134},"详细代码请前往MindSpore论坛进行下载：",{"type":18,"tag":43,"props":136,"children":139},{"href":137,"rel":138},"https://bbs.huaweicloud.com/forum/thread-0252991928222670043-1-1.html",[47],[140],{"type":24,"value":137},{"type":24,"value":52},{"title":7,"searchDepth":143,"depth":143,"links":144},4,[145,147,148],{"id":28,"depth":146,"text":28},3,{"id":55,"depth":146,"text":58},{"id":76,"depth":146,"text":79},"markdown","content:technology-blogs:zh:1866.md","content","technology-blogs/zh/1866.md","technology-blogs/zh/1866","md",1776506116453]