[{"data":1,"prerenderedAt":462},["ShallowReactive",2],{"content-query-8uRf5ICMbQ":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":456,"_id":457,"_source":458,"_file":459,"_stem":460,"_extension":461},"\u002Ftechnology-blogs\u002Fzh\u002F2026-7-9","zh",false,"","MindSpore HyperParallel-Mpipe：多模态流水并行新范式，性能提升1.2X","本文提出 Mpipe，以代数化方式统一刻画流水线调度中的放置、通信与执行顺序。在 512 卡超节点上的典型 MLLM 训练场景中，Mpipe 实现了 1.21× 端到端加速。","2026-7-9","https:\u002F\u002Fobs-mindspore-file.obs.cn-north-4.myhuaweicloud.com\u002Ffile\u002F2024\u002F11\u002F28\u002F8e0e0150508a4c5ba4287fa3bec8ea3f.png","technology-blogs","技术解读",{"type":15,"children":16,"toc":440},"root",[17,25,31,36,43,48,59,64,71,76,101,106,113,118,124,129,134,139,144,149,154,167,172,178,183,188,194,199,204,211,216,221,226,231,238,243,248,254,259,269,274,287,292,297,304,309,315,320,325,330,335,342,347,353,358,363,377,389,395,400,405,414,419,428,433],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"mindspore-hyperparallel-mpipe多模态流水并行新范式性能提升12x",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":28},"p",{},[29],{"type":24,"value":30},"当前 AI 应用已不再局限于纯文本交互。在视觉问答、文档理解、视频分析等真实场景中，模型需要同时理解语言、图像、视频等多模态输入，由此多模态大语言模型 MLLM 已成为工业界关注的重要 方向。然而，现有深度学习框架尚未充分释放超节点在 MLLM 训练中的性能潜力，在 1K–2K GPU 集群上 MFU 仅为 13%–20% 。",{"type":18,"tag":26,"props":32,"children":33},{},[34],{"type":24,"value":35},"为此，本文提出 Mpipe，以代数化方式统一刻画流水线调度中的放置、通信与执行顺序。在 512 卡超节点上的典型 MLLM 训练场景中，Mpipe 实现了 1.21× 端到端加速。",{"type":18,"tag":37,"props":38,"children":40},"h2",{"id":39},"_01-mllm-负载两大根源分析",[41],{"type":24,"value":42},"01 MLLM 负载两大根源分析",{"type":18,"tag":26,"props":44,"children":45},{},[46],{"type":24,"value":47},"MLLM 的模型不再只是一个单一的语言网络。以视觉语言模型为例，图像、视频等输入需先经模态编码器（encoder）提取视觉特征，再经 projector 对齐至 LLM 主干可理解的表示空间，随后由 LLM 主干执行推理；生成式多模态任务中，LLM backbone 的输出还可能继续交由图像、视频或音频生成器（generator），产生对应模态的结果。训练流程亦呈阶段化特征：部分阶段冻结特定模块、仅训练projector，另一些阶段则联合训练更多组件。",{"type":18,"tag":49,"props":50,"children":52},"div",{"style":51},"text-align: center;",[53],{"type":18,"tag":54,"props":55,"children":58},"img",{"src":56,"style":57,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-7-9\u002F1.jpg","display: block;margin: 0 auto;max-width:60%",[],{"type":18,"tag":26,"props":60,"children":61},{},[62],{"type":24,"value":63},"从负载角度看，MLLM 相比 decoder-only LLM 主要引入两类挑战：",{"type":18,"tag":65,"props":66,"children":68},"h3",{"id":67},"挑战一模型结构的模态异构",[69],{"type":24,"value":70},"挑战一：模型结构的模态异构",{"type":18,"tag":26,"props":72,"children":73},{},[74],{"type":24,"value":75},"MLLM 不再执行单一、同质的 Transformer 堆叠，而是将模态解析、特征对齐、语言推理和模态生成拆分为不同功能模块。这些模块各司其职、负载迥异：",{"type":18,"tag":77,"props":78,"children":79},"ul",{},[80,86,91,96],{"type":18,"tag":81,"props":82,"children":83},"li",{},[84],{"type":24,"value":85},"ViT-style encoder：抽取图像\u002F视频的模态特征",{"type":18,"tag":81,"props":87,"children":88},{},[89],{"type":24,"value":90},"projector：将特征对齐到 LLM 的 token 空间",{"type":18,"tag":81,"props":92,"children":93},{},[94],{"type":24,"value":95},"LLM backbone：执行跨模态推理",{"type":18,"tag":81,"props":97,"children":98},{},[99],{"type":24,"value":100},"generator：将推理结果转化为图像\u002F视频\u002F音频",{"type":18,"tag":26,"props":102,"children":103},{},[104],{"type":24,"value":105},"这种异构不仅体现在功能上，也直接体现在计算负载上。模态 encoder 通常比 LLM backbone 窄得多 (模型 Hidden_size)，参数量也远小于 LLM，但它仍可能产生接近 LLM backbone layer的激活显存，并带来不可忽略的计算量。",{"type":18,"tag":49,"props":107,"children":108},{"style":51},[109],{"type":18,"tag":54,"props":110,"children":112},{"src":111,"style":57,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-7-9\u002F2.jpg",[],{"type":18,"tag":26,"props":114,"children":115},{},[116],{"type":24,"value":117},"关键洞察：ViT 编码器的参数量虽小，其激活显存却几乎与 LLM 主干相当，计算量也足以左右流水线调度决策。这些模块并非 Transformer 层的同质副本，而是构成了一条真正异构的模型流水线。",{"type":18,"tag":65,"props":119,"children":121},{"id":120},"挑战二输入数据的动态变化",[122],{"type":24,"value":123},"挑战二：输入数据的动态变化",{"type":18,"tag":26,"props":125,"children":126},{},[127],{"type":24,"value":128},"相比文本，模态内容本身往往更“重”：一段长文本可能仅数 KB，一张高分辨率图片却可达数十 MB，一段高清视频甚至可能达到 GB 级。同时，模态内容高度多变——图像分辨率、视频长度、样本中的图文比例各不相同，甚至有些样本是纯文本。",{"type":18,"tag":26,"props":130,"children":131},{},[132],{"type":24,"value":133},"这些原始模态内容经预处理后，被转换为不同长度的 token 序列。注意力的计算负载和激活内存均与序列长度强相关，因此高分辨率图片或长视频等转换出的长序列，会比小图、短视频等带来更高的 encoder 计算负载。",{"type":18,"tag":26,"props":135,"children":136},{},[137],{"type":24,"value":138},"实际训练中，不仅模态内容占比会变化，模态内容本身的处理开销也会变化。如：视觉语言模型（VLM） 训练常混用高分辨率图片数据集、视频数据集、低分辨率网络图片数据集等，持续引入动态变化的编码器计算负载。",{"type":18,"tag":26,"props":140,"children":141},{},[142],{"type":24,"value":143},"传统 PP 框架的困境\n在 MLLM 的分布式部署中，流水线并行（Pipeline Parallelism, PP）是最常用的扩展方式：模型按层切分到不同设备组（pipeline stage），相邻阶段间传递激活值和梯度。",{"type":18,"tag":26,"props":145,"children":146},{},[147],{"type":24,"value":148},"当前框架的问题在于：一套 PP 计算模式套用全模型。以 VLM 为例，视觉encoder被划入某个流水线阶段，再以 SPMD 方式与 LLM backbone 共用同一种流水线调度。",{"type":18,"tag":26,"props":150,"children":151},{},[152],{"type":24,"value":153},"这种“一套模板包打天下”的做法容易导致两个严重后果：",{"type":18,"tag":77,"props":155,"children":156},{},[157,162],{"type":18,"tag":81,"props":158,"children":159},{},[160],{"type":24,"value":161},"负载不均：encoder与 LLM  backbone 的功能、负载和动态性各不相同，被迫共享同一个流水线模式，就难以像 decoder-only LLM 那样实现均匀阶段划分；",{"type":18,"tag":81,"props":163,"children":164},{},[165],{"type":24,"value":166},"微批次抖动：高分辨率图像或长视频所在的 micro-batch，其 encoder 前向耗时可能从正常的 200–300 ms 骤增至 1000 ms 以上；",{"type":18,"tag":26,"props":168,"children":169},{},[170],{"type":24,"value":171},"这些阶段级别不均衡和 micro-batch 级别的抖动，在流水线中插入大量空泡（bubble），GPU 长期处于等待状态，降低整体训练性能。",{"type":18,"tag":37,"props":173,"children":175},{"id":174},"_02-mpipe-设计代数驱动的可组合调度",[176],{"type":24,"value":177},"02 Mpipe 设计：代数驱动的可组合调度",{"type":18,"tag":26,"props":179,"children":180},{},[181],{"type":24,"value":182},"面对上述问题，Mpipe 没有选择在既有 PP 框架上“打补丁”，而是采用代数驱动视角，对结构化并行的继续推进：BSP将结构化编程的可分析性移植到并行计算领域；Algorithmic Skeletons进一步实现了并行程序的模块化，而OC-P3L则基于函数式骨架将这组合思想升维至代数编程系统。",{"type":18,"tag":26,"props":184,"children":185},{},[186],{"type":24,"value":187},"在我们科研组前辈的肩膀上，Mpipe 进一步把 MLLM 内部不同模块表达成可以组合、可以推导的代数对象，将 MLLM 的 encoder 与 LLM backbone 解耦处理：系统不再把整段模型套进同一个 PP 模板，而是为不同 region 选择合适的并行 skeleton，并通过代数对象组合推导出具体的放置、通信和执行顺序。",{"type":18,"tag":65,"props":189,"children":191},{"id":190},"_21-调度代数从模型结构到-runtime-行为",[192],{"type":24,"value":193},"2.1 调度代数：从模型结构到 runtime 行为",{"type":18,"tag":26,"props":195,"children":196},{},[197],{"type":24,"value":198},"Mpipe 将调度逻辑抽象为可组合、可推导的代数对象。传统流水线并行（PP）以预设模板（如 1F1B）统摄全模型；Mpipe 反之，先辨识模型内计算负载异构的深度区域（depth region），再为各区域独立匹配并行骨架（skeleton），最后由调度代数导出组合后的整体运行行为。",{"type":18,"tag":26,"props":200,"children":201},{},[202],{"type":24,"value":203},"模型定义为层的序列 U = \u003Cu1, ..., uL>；切分（cut）操作将其划分为连续 region（如 VLM 的 encoder 与 backbone）。调度层面上 Mpipe 会进一步会为每个 region 指定一个 skeleton：",{"type":18,"tag":49,"props":205,"children":206},{"style":51},[207],{"type":18,"tag":54,"props":208,"children":210},{"src":209,"style":57,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-7-9\u002F3.jpg",[],{"type":18,"tag":26,"props":212,"children":213},{},[214],{"type":24,"value":215},"上述定义使Mpipe 可将模型切分与并行策略分别描述，并通过推导生成可执行的运行时方案。不同 skeleton 对应不同布局（layout）：流水调度 1F1B 与 GPipe skeleton 对应分片（Sharded）布局，即模型按层切分分布于各流水级（stage）；Transpose skeleton 则对应复制（Replicated）布局，即模型 region 在各流水级上复制。",{"type":18,"tag":26,"props":217,"children":218},{},[219],{"type":24,"value":220},"Seam 规则：代数化的边界连接方案\n这套定义中核心难点在于 region 间的组合：调度的合法性不仅取决于各 region 自身是否可执行，还取决于相邻 region 布局的兼容性及跨 region 数据流的传递方式。",{"type":18,"tag":26,"props":222,"children":223},{},[224],{"type":24,"value":225},"因此可用调度不依赖专家经验枚举，而由两个条件共同判定：切分是否完整覆盖整个模型，以及相邻 skeleton 之间是否存在明确、可推导的连接规则。换言之，Mpipe 不将跨 region 的通信与执行行为硬编码于调度逻辑，而是通过代数规则从 skeleton 的组合关系中推导得出。",{"type":18,"tag":26,"props":227,"children":228},{},[229],{"type":24,"value":230},"我们将相邻 region 间的边界连接规则定义为 seam 规则（seam rule），如表 2 所示。布局决定 region 间的对接方式，seam 规则进一步规定所需的通信操作；这些通信由代数系统自动推导，无需人工编写。",{"type":18,"tag":49,"props":232,"children":233},{"style":51},[234],{"type":18,"tag":54,"props":235,"children":237},{"src":236,"style":57,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-7-9\u002F4.jpg",[],{"type":18,"tag":26,"props":239,"children":240},{},[241],{"type":24,"value":242},"综上所述，Mpipe 推导所得并非静态配置，而是实际 runtime 下可直接执行视图：为各 region 分配执行 rank，推导相邻 region 间的通信及其与计算的执行顺序，表 2 仅为 seam 通信规则的简化示意。",{"type":18,"tag":26,"props":244,"children":245},{},[246],{"type":24,"value":247},"该视图天然蕴含内存行为——执行顺序决定激活值的产生与释放时机，放置决定其所在 rank——因此调度代数同时提供了内存生命周期的分析入口。此外，Mpipe 通过可训练标记（trainable bit）区分不同训练阶段中 region 是否参与反向传播（如 projector-only 对齐阶段与端到端训练阶段），将训练语义与调度方式分离，避免为各训练阶段单独设计调度方案。",{"type":18,"tag":65,"props":249,"children":251},{"id":250},"_22-mpipe-在-vlm-中的调度实例",[252],{"type":24,"value":253},"2.2 Mpipe 在 VLM 中的调度实例",{"type":18,"tag":26,"props":255,"children":256},{},[257],{"type":24,"value":258},"在VLM 场景中，encoder 与 LLM backbone 解耦后的调度为：",{"type":18,"tag":260,"props":261,"children":263},"pre",{"code":262},"\u003Ctranspose, 1f1b>\n",[264],{"type":18,"tag":265,"props":266,"children":267},"code",{"__ignoreMap":7},[268],{"type":24,"value":262},{"type":18,"tag":26,"props":270,"children":271},{},[272],{"type":24,"value":273},"其中encoder region 使用 transpose skeleton，LLM backbone region 使用 1f1b skeleton，代数会推导出一个自然的组合结果：",{"type":18,"tag":77,"props":275,"children":276},{},[277,282],{"type":18,"tag":81,"props":278,"children":279},{},[280],{"type":24,"value":281},"encoder 是“复制 \u002F 以 ByMicrobatch 方式并行计算 micro-batch”的 region；",{"type":18,"tag":81,"props":283,"children":284},{},[285],{"type":24,"value":286},"LLM backbone 是“分片 \u002F 以 ByLayer 方式分配层”的 region；",{"type":18,"tag":26,"props":288,"children":289},{},[290],{"type":24,"value":291},"前向计算中，seam 规则负责将 encoder region 产生的激活值传递至 LLM backbone region，以启动其计算；反向路径则由可训练标记决定。经此组合，encoder 与 LLM backbone 不再被迫共享同一 PP 模板，而是作为两个计算负载各异的独立 region，以各自适配的 skeleton 分别表达，并通过 seam 规则组合拼接。",{"type":18,"tag":26,"props":293,"children":294},{},[295],{"type":24,"value":296},"下面是 Mpipe 推导出的一个具体的实例：",{"type":18,"tag":49,"props":298,"children":299},{"style":51},[300],{"type":18,"tag":54,"props":301,"children":303},{"src":302,"style":57,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-7-9\u002F5.jpg",[],{"type":18,"tag":26,"props":305,"children":306},{},[307],{"type":24,"value":308},"最终，同一组 \u003Ctranspose, 1F1B> 调度可覆盖不同训练阶段，无需为冻结 encoder 与可训练 encoder 分别维护独立的流水线运行时路径。\n在 VLM 下，Mpipe 将 encoder 与 LLM backbone 各自应采用的并行 skeleton，以及二者间的组合方式，统一纳入调度代数的推导框架。因此，从系统集成视角看，Mpipe 并非重写一套 PP runtime，而是一种可组合的多模态流水线优化层：它可在不改动 PP runtime 主体抽象的前提下，对接 1F1B、GPipe 等不同流水线调度变体。",{"type":18,"tag":37,"props":310,"children":312},{"id":311},"_03-实-验",[313],{"type":24,"value":314},"03 实 验",{"type":18,"tag":26,"props":316,"children":317},{},[318],{"type":24,"value":319},"我们在 HyperParallel 中实现 Mpipe，实验运行于昇腾集群上。实验验证了 Mpipe 在真实 MLLM 训练中的性能收益：通过将 encoder 与 LLM backbone 解耦，并以 skeleton 组合的方式为 encoder 应用 transpose、为 LLM backbone 应用 1F1B，Mpipe 能够把多模态模型中的异构计算组织成更适合流水线执行的调度。",{"type":18,"tag":26,"props":321,"children":322},{},[323],{"type":24,"value":324},"实验设置\n我们在生产规模 MLLM 负载上评估 Mpipe，模型部署于 512 卡集群，使用 Qwen2-VL 的 ViT encoder 与 DeepSeek-V3 LLM backbone，数据来自内部多模态数据集。",{"type":18,"tag":26,"props":326,"children":327},{},[328],{"type":24,"value":329},"基线配置：由于 DistTrain 未公开实现，我们在同一环境中构造了类 DistTrain 基线（DistTrain-like baseline）：为 encoder 分配独立流水线 stage 与定制并行策略，LLM backbone 使用常规 5D 并行。",{"type":18,"tag":26,"props":331,"children":332},{},[333],{"type":24,"value":334},"如表 3 所示，Mpipe 将平均单步时间从 16.26 s 降至 13.42 s，取得 1.21× 加速。",{"type":18,"tag":49,"props":336,"children":337},{"style":51},[338],{"type":18,"tag":54,"props":339,"children":341},{"src":340,"style":57,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-7-9\u002F6.jpg",[],{"type":18,"tag":26,"props":343,"children":344},{},[345],{"type":24,"value":346},"上述结果表明，Mpipe 的代数建模与 skeleton 组合能够带来实际的端到端收益。实验中 LLM backbone 体量远超 ViT encoder，整体单步时间仍由 backbone 主导，在此前提下 1.21× 的端到端加速已对应集群级训练效率的显著提升。Mpipe 不引入每轮迭代的额外调度开销，亦不改变训练 loss 语义，仅通过 skeleton 组合重新组织 encoder 与 LLM backbone 间的执行关系。",{"type":18,"tag":37,"props":348,"children":350},{"id":349},"_04-总-结",[351],{"type":24,"value":352},"04 总 结",{"type":18,"tag":26,"props":354,"children":355},{},[356],{"type":24,"value":357},"Mpipe 以调度代数对 MLLM 并行进行符号化建模，将模型结构映射为具体的流水线行为。通过 \u003Ctranspose, 1f1b> 骨架组合，Mpipe 推导出面向 MLLM 的异构并行调度：编码器以复制方式填充流水线 warmup 空泡，LLM 主干以常规流水线骨架执行，在保持训练语义不变的前提下，将模态异构与动态输入引入的不规则负载转化为可组合、可调度的执行计划，最终带来端到端性能提升。",{"type":18,"tag":26,"props":359,"children":360},{},[361],{"type":24,"value":362},"Mpipe 原型已在 512 卡生产规模负载上验证 1.21× 加速能力。随着 MLLM 引入更多模态、更长上下文与更复杂的 encoder\u002Fgenerator 组合，模型异构性与动态性将持续增强，Mpipe 基于代数的组合调度将具备更大的优化空间。",{"type":18,"tag":26,"props":364,"children":365},{},[366,368],{"type":24,"value":367},"使用方式\nMpipe 已集成至 HyperParallel 框架，作为内置调度策略接入昇腾集群。具体调用方式参见：（",{"type":18,"tag":369,"props":370,"children":374},"a",{"href":371,"rel":372},"https:\u002F\u002Fatomgit.com\u002Fmindspore\u002Fhyper-parallel\u002Fblob\u002Fmaster\u002Fdocs\u002Fguide\u002Fpipeline_parallel.md%EF%BC%89",[373],"nofollow",[375],{"type":24,"value":376},"https:\u002F\u002Fatomgit.com\u002Fmindspore\u002Fhyper-parallel\u002Fblob\u002Fmaster\u002Fdocs\u002Fguide\u002Fpipeline_parallel.md）",{"type":18,"tag":26,"props":378,"children":379},{},[380,382],{"type":24,"value":381},"关于Mpipe的详细设计思路与更多实验分析，可阅读完整论文（",{"type":18,"tag":369,"props":383,"children":386},{"href":384,"rel":385},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.03229%EF%BC%89",[373],[387],{"type":24,"value":388},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.03229）",{"type":18,"tag":37,"props":390,"children":392},{"id":391},"_05-欢迎加入-mindspore-hyperparallel",[393],{"type":24,"value":394},"05 欢迎加入 MindSpore HyperParallel",{"type":18,"tag":26,"props":396,"children":397},{},[398],{"type":24,"value":399},"MindSpore HyperParallel 致力于保障分布式训练的稳定性与精度可复现，这离不开每一位开发者的智慧。如果你也追求极致的算力释放，欢迎加入我们的 SIG（特别兴趣小组）：",{"type":18,"tag":26,"props":401,"children":402},{},[403],{"type":24,"value":404},"参与讨论：加入 Parallel Training System SIG 论坛",{"type":18,"tag":26,"props":406,"children":407},{},[408],{"type":18,"tag":369,"props":409,"children":412},{"href":410,"rel":411},"https:\u002F\u002Fwww.mindspore.cn\u002Fsig\u002FParallel%20Training%20System",[373],[413],{"type":24,"value":410},{"type":18,"tag":26,"props":415,"children":416},{},[417],{"type":24,"value":418},"代码共建：在 GitCode 提交 Issue 或 Pull Request",{"type":18,"tag":26,"props":420,"children":421},{},[422],{"type":18,"tag":369,"props":423,"children":426},{"href":424,"rel":425},"https:\u002F\u002Fgitcode.com\u002Fmindspore\u002Fhyper-parallel\u002F",[373],[427],{"type":24,"value":424},{"type":18,"tag":26,"props":429,"children":430},{},[431],{"type":24,"value":432},"无论是代码实现、文档完善、示例补充还是Bug 反馈，您的每一份贡献都将帮助更多研究者和开发者受益。",{"type":18,"tag":49,"props":434,"children":435},{"style":51},[436],{"type":18,"tag":54,"props":437,"children":439},{"src":438,"style":57,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-7-9\u002F7.jpg",[],{"title":7,"searchDepth":441,"depth":441,"links":442},4,[443,449,453,454,455],{"id":39,"depth":444,"text":42,"children":445},2,[446,448],{"id":67,"depth":447,"text":70},3,{"id":120,"depth":447,"text":123},{"id":174,"depth":444,"text":177,"children":450},[451,452],{"id":190,"depth":447,"text":193},{"id":250,"depth":447,"text":253},{"id":311,"depth":444,"text":314},{"id":349,"depth":444,"text":352},{"id":391,"depth":444,"text":394},"markdown","content:technology-blogs:zh:2026-7-9.md","content","technology-blogs\u002Fzh\u002F2026-7-9.md","technology-blogs\u002Fzh\u002F2026-7-9","md",1784755783059]