[{"data":1,"prerenderedAt":241},["ShallowReactive",2],{"content-query-fNcilJqxkq":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":235,"_id":236,"_source":237,"_file":238,"_stem":239,"_extension":240},"\u002Ftechnology-blogs\u002Fzh\u002F2026-9-18","zh",false,"","MindSpore  HyperParallel一站式使能HuggingFace模型高性能多维并行","在保留HuggingFace模型生态和使用习惯的基础上，通过声明式配置实现性能优化和多维并行能力，降低模型走向高性能分布式训练的适配成本。","2026-9-18","https:\u002F\u002Fobs-mindspore-file.obs.cn-north-4.myhuaweicloud.com\u002Ffile\u002F2024\u002F11\u002F28\u002F8e0e0150508a4c5ba4287fa3bec8ea3f.png","technology-blogs","技术解读",{"type":15,"children":16,"toc":232},"root",[17,25,36,41,48,55,60,65,70,80,85,91,96,101,109,116,122,127,134,170,178,184,194,200,210,224],{"type":18,"tag":19,"props":20,"children":21},"element","p",{},[22],{"type":23,"value":24},"text","HuggingFace凭借丰富的模型生态和统一的接口，让开发者能够更方便地获取和使用各类大模型。但在大规模训练场景下，如何进一步提升训练性能、使能FSDP、TP、CP、EP等多维并行能力，仍然需要开发者进行额外适配。\nHyperParallel希望解决的正是这一问题：在保留HuggingFace模型生态和使用习惯的基础上，通过声明式配置实现性能优化和多维并行能力，降低模型走向高性能分布式训练的适配成本。",{"type":18,"tag":26,"props":27,"children":29},"div",{"style":28},"text-align: center;",[30],{"type":18,"tag":31,"props":32,"children":35},"img",{"src":33,"style":34,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-9-18\u002F1.jpg","display: block;margin: 0 auto;max-width:60%",[],{"type":18,"tag":19,"props":37,"children":38},{},[39],{"type":23,"value":40},"在近日举办的PyTorch Conference China 2026上，我们也围绕HyperParallel的整体设计与实践进行了分享，重点介绍了其在HuggingFace模型兼容、高性能模块替换以及多维并行等方面的最新进展。",{"type":18,"tag":26,"props":42,"children":43},{"style":28},[44],{"type":18,"tag":31,"props":45,"children":47},{"src":46,"style":34,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-9-18\u002F2.jpg",[],{"type":18,"tag":49,"props":50,"children":52},"h1",{"id":51},"_01-huggingface模型原生适配",[53],{"type":23,"value":54},"01 HuggingFace模型原生适配",{"type":18,"tag":19,"props":56,"children":57},{},[58],{"type":23,"value":59},"传统高性能分布式训练方案中，新模型往往需要重新实现高性能模块，再处理参数映射、权重加载和并行逻辑等适配工作。模型一旦升级，适配成本也随之增加。",{"type":18,"tag":19,"props":61,"children":62},{},[63],{"type":23,"value":64},"HyperParallel采用另一条路径：直接复用HuggingFace模型结构、Config和预训练权重，并在模型构建过程中注入系统优化能力。",{"type":18,"tag":19,"props":66,"children":67},{},[68],{"type":23,"value":69},"以当前官方示例为例，用户仍然通过HuggingFace模型路径加载模型：",{"type":18,"tag":71,"props":72,"children":74},"pre",{"code":73},"model:\n  _target_: hyper_parallel.models.HyperAutoModelForCausalLM.from_pretrained\n  pretrained_model_name_or_path: \u002Fpath\u002Fto\u002FQwen3-30B-A3B\n  torch_dtype: bfloat16\n  force_hf: true\n",[75],{"type":18,"tag":76,"props":77,"children":78},"code",{"__ignoreMap":7},[79],{"type":23,"value":73},{"type":18,"tag":19,"props":81,"children":82},{},[83],{"type":23,"value":84},"用户不需要为了实现大规模分布式训练，重新维护一套模型。HuggingFace继续负责模型生态，HyperParallel负责系统优化。",{"type":18,"tag":49,"props":86,"children":88},{"id":87},"_02-高性能模块声明式替换",[89],{"type":23,"value":90},"02 高性能模块声明式替换",{"type":18,"tag":19,"props":92,"children":93},{},[94],{"type":23,"value":95},"通用实现不一定是目标硬件上的最优实现。针对Attention、Norm、MoE等关键计算模块，HyperParallel支持在保留原有HuggingFace模型结构的基础上，按需替换为高性能实现。",{"type":18,"tag":19,"props":97,"children":98},{},[99],{"type":23,"value":100},"用户无需修改模型源码，只需要通过 plan_overrides 描述匹配哪个Module，以及替换成什么实现：",{"type":18,"tag":71,"props":102,"children":104},{"code":103},"plan_overrides:  \n - match: \"*.self_attn\"    \n   module_type: transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeAttention    \n   replace_module:      \n     _target_: hyper_parallel.models.qwen3_moe.adapter.replacements.replace_qwen3_moe_flash_attention\n以Qwen3-30B-A3B为例，当前示例包含以下高性能模块替换：\n",[105],{"type":18,"tag":76,"props":106,"children":107},{"__ignoreMap":7},[108],{"type":23,"value":103},{"type":18,"tag":26,"props":110,"children":111},{"style":28},[112],{"type":18,"tag":31,"props":113,"children":115},{"src":114,"style":34,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-9-18\u002F3.jpg",[],{"type":18,"tag":49,"props":117,"children":119},{"id":118},"_03-多维并行能力声明式配置",[120],{"type":23,"value":121},"03 多维并行能力声明式配置",{"type":18,"tag":19,"props":123,"children":124},{},[125],{"type":23,"value":126},"完成模型接入和高性能模块替换后，接下来需要考虑的是如何将模型合理切分到更多设备上，实现高效的大规模训练。FSDP、TP、CP、EP分别对应四类最常见的训练瓶颈。",{"type":18,"tag":26,"props":128,"children":129},{"style":28},[130],{"type":18,"tag":31,"props":131,"children":133},{"src":132,"style":34,"alt":7},"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-9-18\u002F4.jpg",[],{"type":18,"tag":19,"props":135,"children":136},{},[137,139,143,145,148,150,153,155,158,160,163,165,168],{"type":23,"value":138},"fsdp_config:",{"type":18,"tag":140,"props":141,"children":142},"br",{},[],{"type":23,"value":144},"\ndp_shard_size: 2",{"type":18,"tag":140,"props":146,"children":147},{},[],{"type":23,"value":149},"\nedp_shard_size: 2",{"type":18,"tag":140,"props":151,"children":152},{},[],{"type":23,"value":154},"\nreshard_after_forward: true",{"type":18,"tag":140,"props":156,"children":157},{},[],{"type":23,"value":159},"\nreshard_after_backward: true",{"type":18,"tag":140,"props":161,"children":162},{},[],{"type":23,"value":164},"\nforward_prefetch_depth: 1",{"type":18,"tag":140,"props":166,"children":167},{},[],{"type":23,"value":169},"\nbackward_prefetch_depth: 1",{"type":18,"tag":71,"props":171,"children":173},{"code":172},"以上为8卡多维并行配置示例，通过TP=2、CP=2、EP=2结合FSDP完成模型切分。\n\n# 04 双模式DTensor：兼顾性能与正确性验证\n\n声明式并行越自动，正确性越重要。Shard维度选错、Module边界Layout不匹配、Redistribute遗漏，都可能让多卡语义与原始模型产生偏差，而且不一定马上报错。\n\n但真实训练又不希望每一步都承担额外的DTensor Dispatch和动态Layout推导开销。HyperParallel因此将这两个目标拆成Production和Validation两种模式。\n\n\u003Cdiv style=\"text-align: center;\">\n\n\u003Cimg src=\"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-9-18\u002F5.jpg\" style=\"display: block;margin: 0 auto;max-width:60%\" alt=\"\" >\n\n\u003C\u002Fdiv>\n真正关键的不是“有两个模式”，而是验证路径不能成为另一套实现。两种模式共享同一份Sharding Plan。对于CP Attention、MoE Token Dispatch等这类需要显式通信的场景，Production和Validation也会尽量复用相同的执行逻辑，保证验证结果能够真实反映生产环境。\n\nProduction降低DTensor动态调度带来的运行时开销，Validation用于验证并行策略及分布式布局的正确性。\n\n# 05 统一配置驱动高性能分布式训练\n\n\u003Cdiv style=\"text-align: center;\">\n\n\u003Cimg src=\"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-9-18\u002F6.jpg\" style=\"display: block;margin: 0 auto;max-width:60%\" alt=\"\" >\n\n\u003C\u002Fdiv>\n\n过去散落在模型代码、并行代码和硬件优化代码里的工作，被重新组织成几个更简单的问题：\n\n1.模型从哪里来？—— HuggingFace生态\n2.模型怎么跑得更快？—— High Performance Module\n3.模型怎么切？—— FSDP \u002F TP \u002F CP \u002F EP\n4.怎么兼顾性能和正确性？—— Production \u002F Validation\n\nHyperParallel想做的，不只是增加几个分布式API。\n\n它更希望让高性能分布式训练，从“模型必须主动适配”逐渐变成“围绕模型声明配置”。从“为了分布式训练重新实现模型”，走向“保留一份HuggingFace模型，声明它应该如何高效运行”。\n\n# 06 快速上手：一键体验多维并行\n\nHyperParallel提供examples\u002Ftraining_demo用于快速体验FSDP、TP、CP、EP等多维并行能力。Qwen3-30B-A3B完整模型建议在满足模型显存和并行配置要求的集群环境下运行；如果暂时缺少相应集群资源，也可以通过裁剪模型层数等配置，在单机8卡环境下快速跑通多维并行训练流程：\n",[174],{"type":18,"tag":76,"props":175,"children":176},{"__ignoreMap":7},[177],{"type":23,"value":172},{"type":18,"tag":49,"props":179,"children":181},{"id":180},"offline数据模式",[182],{"type":23,"value":183},"Offline数据模式",{"type":18,"tag":19,"props":185,"children":186},{},[187,189,192],{"type":23,"value":188},"bash examples\u002Ftraining_demo\u002Frun_parallel_offline.sh \\",{"type":18,"tag":140,"props":190,"children":191},{},[],{"type":23,"value":193},"\n\u002Fpath\u002Fto\u002FQwen3-30B-A3B",{"type":18,"tag":49,"props":195,"children":197},{"id":196},"online数据模式",[198],{"type":23,"value":199},"Online数据模式",{"type":18,"tag":19,"props":201,"children":202},{},[203,205,208],{"type":23,"value":204},"bash examples\u002Ftraining_demo\u002Frun_parallel_online.sh \\",{"type":18,"tag":140,"props":206,"children":207},{},[],{"type":23,"value":209},"\n\u002Fpath\u002Fto\u002FQwen3-30B-A3B\n还可以直接开启DTensor Placement Validation：",{"type":18,"tag":19,"props":211,"children":212},{},[213,214,217,219,222],{"type":23,"value":188},{"type":18,"tag":140,"props":215,"children":216},{},[],{"type":23,"value":218},"\n\u002Fpath\u002Fto\u002FQwen3-30B-A3B \\",{"type":18,"tag":140,"props":220,"children":221},{},[],{"type":23,"value":223},"\n--model.validate_placement=true",{"type":18,"tag":71,"props":225,"children":227},{"code":226},"完整示例可参考：\n\nhttps:\u002F\u002Fgithub.com\u002Fmindspore-ai\u002Fhyper-parallel\u002Ftree\u002Fmaster\u002Fexamples\u002Ftraining_demo\n\n# 07 欢迎加入 MindSpore HyperParallel\n\n我们诚挚邀请各位开发者、研究者加入HyperParallel。无论是贡献代码、完善文档，还是提出改进建议，您的参与都将推动大模型分布式并行技术的边界。让我们一起，让大模型训练更简单、更快速、更智能！\n\n\u003Cdiv style=\"text-align: center;\">\n\n\u003Cimg src=\"\u002Fcategory\u002Finformation\u002Ftechnology-blogs\u002Fbanner\u002F2026-9-18\u002F7.jpg\" style=\"display: block;margin: 0 auto;max-width:60%\" alt=\"\" >\n\n\u003C\u002Fdiv>\n无论是代码实现、文档完善、示例补充还是Bug反馈，您的每一份贡献都将帮助更多研究者和开发者受益。\n\n",[228],{"type":18,"tag":76,"props":229,"children":230},{"__ignoreMap":7},[231],{"type":23,"value":226},{"title":7,"searchDepth":233,"depth":233,"links":234},4,[],"markdown","content:technology-blogs:zh:2026-9-18.md","content","technology-blogs\u002Fzh\u002F2026-9-18.md","technology-blogs\u002Fzh\u002F2026-9-18","md",1789904230868]