[{"data":1,"prerenderedAt":168},["ShallowReactive",2],{"content-query-Wrhvl0jZte":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":162,"_id":163,"_source":164,"_file":165,"_stem":166,"_extension":167},"/technology-blogs/zh/1395","zh",false,"","Pytorch 1.11看上去很美，但是.....","Pytorch如何解决既要维护当前开发者的开发习惯又要充分发挥越来越DSA的硬件性能的矛盾","2022-03-29","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2022/03/29/2cb45e9b9f7d4315813028ee630fc879.jpg","technology-blogs","大V博文",{"type":15,"children":16,"toc":159},"root",[17,25,31,44,49,59,69,74,99,104,109,114,119,129,134,144,149,154],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"pytorch-111看上去很美但是",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":28},"p",{},[29],{"type":24,"value":30},"作者：金雪锋",{"type":18,"tag":26,"props":32,"children":33},{},[34,36],{"type":24,"value":35},"来源：",{"type":18,"tag":37,"props":38,"children":42},"a",{"href":39,"rel":40},"https://zhuanlan.zhihu.com/p/488413600",[41],"nofollow",[43],{"type":24,"value":39},{"type":18,"tag":26,"props":45,"children":46},{},[47],{"type":24,"value":48},"Pytorch团队是一个非常善于学习的团队，从动态图起家，不停在吸收其他框架的优点，比如最新的1.11版本，提供对标JAX的function graph、非常类似TF/MindSpore的TorchData等。",{"type":18,"tag":26,"props":50,"children":51},{},[52],{"type":18,"tag":37,"props":53,"children":56},{"href":54,"rel":55},"https://link.zhihu.com/?target=https%3A//github.com/pytorch/pytorch/releases/tag/v1.11.0",[41],[57],{"type":24,"value":58},"https://github.com/pytorch/pytorch/releases/tag/v1.11.0",{"type":18,"tag":26,"props":60,"children":61},{},[62],{"type":18,"tag":37,"props":63,"children":66},{"href":64,"rel":65},"https://zhuanlan.zhihu.com/p/480496499",[41],[67],{"type":24,"value":68},"终于可用可组合函数转换库！PyTorch 1.11发布，弥补JAX短板，支持Python 3.10",{"type":18,"tag":26,"props":70,"children":71},{},[72],{"type":24,"value":73},"我们先看看对标JAX的function graph，大家知道，JAX的设计思路是Numpy/Scipy这些标准的Python科学计算库上提供自动微分/分布式并行/jit等扩展，包括：",{"type":18,"tag":75,"props":76,"children":77},"ul",{},[78,84,89,94],{"type":18,"tag":79,"props":80,"children":81},"li",{},[82],{"type":24,"value":83},"自动微分：vjp/jvp/grad/......",{"type":18,"tag":79,"props":85,"children":86},{},[87],{"type":24,"value":88},"分布式：pmap/.....",{"type":18,"tag":79,"props":90,"children":91},{},[92],{"type":24,"value":93},"向量化并行：vmap/....",{"type":18,"tag":79,"props":95,"children":96},{},[97],{"type":24,"value":98},"JIT加速",{"type":18,"tag":26,"props":100,"children":101},{},[102],{"type":24,"value":103},"从特性上看，最关键是自动微分和分布式这两块",{"type":18,"tag":26,"props":105,"children":106},{},[107],{"type":24,"value":108},"从架构上看，JAX采用的是函数式风格，允许上面的接口通过闭包等形式组合使用。",{"type":18,"tag":26,"props":110,"children":111},{},[112],{"type":24,"value":113},"而这次Pytorch提供的function graph，缺失了最重要的pmap接口，因为Pytorch的静态图不支持分布式并行；",{"type":18,"tag":26,"props":115,"children":116},{},[117],{"type":24,"value":118},"另外，个人认为最重要的问题是，function graph所提供的函数式编程风格其实和Pytorch nn的Tensor为中心的风格是不统一的，这一点在自动微分的时候，尤其明显，容易让开发者感到困惑，参考：",{"type":18,"tag":26,"props":120,"children":121},{},[122],{"type":18,"tag":37,"props":123,"children":126},{"href":124,"rel":125},"https://zhuanlan.zhihu.com/p/393031067",[41],[127],{"type":24,"value":128},"聊一聊AI框架前端",{"type":18,"tag":26,"props":130,"children":131},{},[132],{"type":24,"value":133},"在看一下TorchData，前几天团队的小伙伴们还在说，Torch的Dataloader比较灵活，易用性好，结果没几天，就被打脸了，TorchData就出现了，居然和MindSpore的设计类似了，参考：",{"type":18,"tag":26,"props":135,"children":136},{},[137],{"type":18,"tag":37,"props":138,"children":141},{"href":139,"rel":140},"https://zhuanlan.zhihu.com/p/352487023",[41],[142],{"type":24,"value":143},"AI框架中数据处理的挑战与解决思路",{"type":18,"tag":26,"props":145,"children":146},{},[147],{"type":24,"value":148},"实际上，随着训练数据的越来越多，模型的越来越大，数据的处理已经成为大规模训练的瓶颈，我个人感觉TorchData还是改造的不够彻底，大量的处理还是依赖Python，后面还会是瓶颈。",{"type":18,"tag":26,"props":150,"children":151},{},[152],{"type":24,"value":153},"回溯Pytorch的历史，其实从动态图取得优势后，Pytorch做了很多工作，比如torchscript、tracing、FX等等，但是，我们也看到这些工作基本上是作为动态图的补充，我想这也可能是function graph和torch data设计成这样的根因。",{"type":18,"tag":26,"props":155,"children":156},{},[157],{"type":24,"value":158},"面向未来，我更期待是，随着NV H100以及更多的AI芯片发布，Pytorch如何解决既要维护当前开发者的开发习惯又要充分发挥越来越DSA的硬件性能的矛盾。",{"title":7,"searchDepth":160,"depth":160,"links":161},4,[],"markdown","content:technology-blogs:zh:1395.md","content","technology-blogs/zh/1395.md","technology-blogs/zh/1395","md",1776506112196]