[{"data":1,"prerenderedAt":483},["ShallowReactive",2],{"content-query-x0XMk8aAP3":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":477,"_id":478,"_source":479,"_file":480,"_stem":481,"_extension":482},"/technology-blogs/zh/3651","zh",false,"","全流程加速MoE模型预训练及强化学习！揭秘昇思MindSpore并行技术","为强化学习开发者提供了训练接口，支持算法快速开发，提供多种训练优化技术。","2025-03-18","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/26/642cee8d2668420fa8a651bc45024dba.png","technology-blogs","实践",{"type":15,"children":16,"toc":474},"root",[17,25,31,36,50,55,75,80,88,93,98,105,110,115,122,127,132,141,149,157,164,179,187,192,202,207,215,227,235,256,264,269,277,281,289,297,305,310,318,323,331,339,343,351,359,367,372,380,385,390,398,406,411,419,423,431,436,441,449,453,461,466],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"全流程加速moe模型预训练及强化学习揭秘昇思mindspore并行技术",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":28},"p",{},[29],{"type":24,"value":30},"昇思MindSpore已实现DeepSeek V3 + GRPO强化学习训练全流程支持，并将相关代码开源，为强化学习开发者提供了训练接口，支持算法快速开发，提供多种训练优化技术。",{"type":18,"tag":26,"props":32,"children":33},{},[34],{"type":24,"value":35},"当前代码已在昇思社区开源，根据本教程，用户可以快速上手体验，或尝试完成DeepSeek V3 R1的训练全过程。",{"type":18,"tag":26,"props":37,"children":38},{},[39,41],{"type":24,"value":40},"开源代码仓地址：",{"type":18,"tag":42,"props":43,"children":47},"a",{"href":44,"rel":45},"https://gitee.com/mindspore/mindrlhf/tree/master/examples/grpo/deepseekv3%5C_grpo%5C_tutorial",[46],"nofollow",[48],{"type":24,"value":49},"https://gitee.com/mindspore/mindrlhf/tree/master/examples/grpo/deepseekv3\\_grpo\\_tutorial",{"type":18,"tag":26,"props":51,"children":52},{},[53],{"type":24,"value":54},"昇思MindSpore框架为DeepSeek V3模型训练流程提供了以下性能优化技术：",{"type":18,"tag":56,"props":57,"children":58},"ul",{},[59,65,70],{"type":18,"tag":60,"props":61,"children":62},"li",{},[63],{"type":24,"value":64},"专家并行下机间零冗余通信技术和共享专家通信掩盖结合，大幅减少超细粒度专家并行带来的通信时间开销。",{"type":18,"tag":60,"props":66,"children":67},{},[68],{"type":24,"value":69},"自动专家负载均衡auxloss free训练技术，解决模型专家负载均衡问题并提高训练稳定性。",{"type":18,"tag":60,"props":71,"children":72},{},[73],{"type":24,"value":74},"流水线并行1B1F 前反向掩盖技术，提高流水线并行场景下训练速度。",{"type":18,"tag":26,"props":76,"children":77},{},[78],{"type":24,"value":79},"MindSpore实现了DP/TP/PP/CP/Zero/EP多维混合并行模式下的权重在线重排，支持DeepSeek V3模型在GRPO强化学习训练与推理阶段自由选择不同并行策略，并且在线进行训推权重转换。其中，在训练阶段支持数据并行（DP），张量并行（TP），流水线并行（PP），专家并行（EP），序列并行（SP）和ZeRO优化器并行；在推理阶段支持前端数据并行（DP）和张量并行（TP）。",{"type":18,"tag":26,"props":81,"children":82},{},[83],{"type":18,"tag":84,"props":85,"children":87},"img",{"alt":7,"src":86},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/26/08d9ce08b1304c55a26f663b2fb81108.png",[],{"type":18,"tag":26,"props":89,"children":90},{},[91],{"type":24,"value":92},"不久之前，昇思MindSpore携手鹏城实验室发布并开源了基于Qwen2.5（7B，32B）模型的GRPO强化学习训练全流程和代码，实现了组件化解耦训练流程与模型定义；升级了MindSpore技术架构，支持训推共部署，训练和推理权重在线快速自动重排与异构内存Swap等技术，成功构建从硬件算力、算法优化到集群调度的完整技术链条。",{"type":18,"tag":26,"props":94,"children":95},{},[96],{"type":24,"value":97},"本次更进一步，MindSpore基于DeepSeek V3模型打通GRPO强化学习训练全流程。与Qwen2.5（7B，32B）模型相比，DeepSeek V3 模型采用超细粒度专家MoE结构，带来了训推过程中模型专家负载均衡的问题，以及训推权重在线快速重排中对专家并行（EP）切分策略的支持的新挑战。同时，超大的模型规模对训练性能和显存管理也提出了更高的要求。",{"type":18,"tag":26,"props":99,"children":100},{},[101],{"type":18,"tag":84,"props":102,"children":104},{"alt":7,"src":103},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/26/026b0ab14add4c9b93490a2f37bc3b84.png",[],{"type":18,"tag":26,"props":106,"children":107},{},[108],{"type":24,"value":109},"DeepSeek V3模型通过超稀疏MoE架构与多线性注意力（MLA）的协同创新，实现了性能与效率的双重突破，使模型推理成本较同性能闭源模型下降90%以上。此外，多标记预测（MTP）机制突破传统单token预测限制，在提升训练效率3倍的同时，生成文本的连贯性达到人类水平。",{"type":18,"tag":26,"props":111,"children":112},{},[113],{"type":24,"value":114},"这些创新使DeepSeek V3在数学推理、代码生成等任务中超越GPT-4等闭源模型，开创了开源模型性能与成本平衡的新范式。",{"type":18,"tag":26,"props":116,"children":117},{},[118],{"type":18,"tag":84,"props":119,"children":121},{"alt":7,"src":120},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/26/144b66b1e69a496fbda2a55b2419de67.png",[],{"type":18,"tag":26,"props":123,"children":124},{},[125],{"type":24,"value":126},"GRPO（Group Relative Policy Optimization，组相对策略优化）是针对数学等逻辑推理任务提出的强化学习训练的算法。强化学习的训练过程是学习一个策略模型，通过不断试错，策略模型与奖励函数的不断交互，策略模型会逐渐倾向于选择能获得更高奖励的行为，自主探索出最佳学习路径。",{"type":18,"tag":26,"props":128,"children":129},{},[130],{"type":24,"value":131},"以DeepSeek V3作为基础模型，通过GRPO算法的大规模强化学习后训练得到的DeepSeek R1模型在逻辑推理能力上得到了显著提升，涌现出了长思维链和反思等深度思考能力。",{"type":18,"tag":26,"props":133,"children":134},{},[135],{"type":18,"tag":136,"props":137,"children":138},"strong",{},[139],{"type":24,"value":140},"# 01****基于DeepSeek V3的GRPO强化学习训练教程",{"type":18,"tag":26,"props":142,"children":143},{},[144],{"type":18,"tag":136,"props":145,"children":146},{},[147],{"type":24,"value":148},"一****环境搭建",{"type":18,"tag":26,"props":150,"children":151},{},[152],{"type":18,"tag":136,"props":153,"children":154},{},[155],{"type":24,"value":156},"1、版本匹配关系",{"type":18,"tag":26,"props":158,"children":159},{},[160],{"type":18,"tag":84,"props":161,"children":163},{"alt":7,"src":162},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/26/bff846fc89fd4fc591ed960168173871.png",[],{"type":18,"tag":26,"props":165,"children":166},{},[167,172,174],{"type":18,"tag":136,"props":168,"children":169},{},[170],{"type":24,"value":171},"二",{"type":24,"value":173}," ",{"type":18,"tag":136,"props":175,"children":176},{},[177],{"type":24,"value":178},"使用指南",{"type":18,"tag":26,"props":180,"children":181},{},[182],{"type":18,"tag":136,"props":183,"children":184},{},[185],{"type":24,"value":186},"1、数据集及文件的获取",{"type":18,"tag":26,"props":188,"children":189},{},[190],{"type":24,"value":191},"使用examples/grpo/deepseekv3_grpo_tutorial/rlhf_data.py将GSM8k.json转换成mindrecord的形式，此数据路径为mind_dataset_dir的取值。此数据路径在启动训推作为mind_dataset_dir 的值。",{"type":18,"tag":193,"props":194,"children":196},"pre",{"code":195},"python rlhf_data.py –tokenizer_file /path/to/tokenizer.json --file_path /path/to/raw/data/ --output_path /path/to/mindrecord/\n",[197],{"type":18,"tag":198,"props":199,"children":200},"code",{"__ignoreMap":7},[201],{"type":24,"value":195},{"type":18,"tag":26,"props":203,"children":204},{},[205],{"type":24,"value":206},"参数说明",{"type":18,"tag":193,"props":208,"children":210},{"code":209},"tokenizer_file：tokenizer.json路径\nfile_path：原始数据文件路径\noutput_path：输出文件路径\n",[211],{"type":18,"tag":198,"props":212,"children":213},{"__ignoreMap":7},[214],{"type":24,"value":209},{"type":18,"tag":26,"props":216,"children":217},{},[218,220],{"type":24,"value":219},"其中tokenizer.json文件可以从HuggingFace（",{"type":18,"tag":42,"props":221,"children":224},{"href":222,"rel":223},"https://huggingface.co/deepseek-ai/DeepSeek-V3%EF%BC%89%E5%AE%98%E6%96%B9%E8%8E%B7%E5%8F%96%E3%80%82",[46],[225],{"type":24,"value":226},"https://huggingface.co/deepseek-ai/DeepSeek-V3）官方获取。",{"type":18,"tag":26,"props":228,"children":229},{},[230],{"type":18,"tag":136,"props":231,"children":232},{},[233],{"type":24,"value":234},"2、权重获取",{"type":18,"tag":26,"props":236,"children":237},{},[238,240,247,249],{"type":24,"value":239},"用户可以从HuggingFace官方（",{"type":18,"tag":42,"props":241,"children":244},{"href":242,"rel":243},"https://huggingface.co/deepseek-ai/DeepSeek-V3%EF%BC%89%E4%B8%8B%E8%BD%BD%E9%A2%84%E8%AE%AD%E7%BB%83%E6%9D%83%E9%87%8D%EF%BC%8C%E7%BB%8F%E8%BF%87%E6%A8%A1%E5%9E%8B%E6%9D%83%E9%87%8D%E8%BD%AC%E6%8D%A2%E4%BB%A5%E5%8F%8A%E5%88%86%E5%B8%83%E5%BC%8F%E6%9D%83%E9%87%8D%E5%88%87%E5%88%86%E5%90%8E%E8%BF%9B%E8%A1%8C%E4%BD%BF%E7%94%A8%EF%BC%8C",[46],[245],{"type":24,"value":246},"https://huggingface.co/deepseek-ai/DeepSeek-V3）下载预训练权重，经过模型权重转换以及分布式权重切分后进行使用，",{"type":24,"value":248}," 可以参考DeepSeek V3（",{"type":18,"tag":42,"props":250,"children":253},{"href":251,"rel":252},"https://gitee.com/mindspore/mindformers/tree/dev/research/deepseek3%EF%BC%89%E8%BF%9B%E8%A1%8C%E6%93%8D%E4%BD%9C%E3%80%82",[46],[254],{"type":24,"value":255},"https://gitee.com/mindspore/mindformers/tree/dev/research/deepseek3）进行操作。",{"type":18,"tag":26,"props":257,"children":258},{},[259],{"type":18,"tag":136,"props":260,"children":261},{},[262],{"type":24,"value":263},"2.1 MindSpore权重转换",{"type":18,"tag":26,"props":265,"children":266},{},[267],{"type":24,"value":268},"完整权重转为MindSpore用的ckpt，进入MindSpore Formers路径下",{"type":18,"tag":193,"props":270,"children":272},{"code":271},"python research/deepseek3/convert_weight.py --torch_ckpt_path TORCH_CKPT_DIR --mindspore_ckpt_path {path}/MS_CKPT_NAME --dtype bf16\n",[273],{"type":18,"tag":198,"props":274,"children":275},{"__ignoreMap":7},[276],{"type":24,"value":271},{"type":18,"tag":26,"props":278,"children":279},{},[280],{"type":24,"value":206},{"type":18,"tag":193,"props":282,"children":284},{"code":283},"torch_ckpt_path:  下载HuggingFace权重的文件夹路径\noutput_path:      转换后的MindSpore权重文件保存路径\ndtype:            转换权重的精度\n",[285],{"type":18,"tag":198,"props":286,"children":287},{"__ignoreMap":7},[288],{"type":24,"value":283},{"type":18,"tag":26,"props":290,"children":291},{},[292],{"type":18,"tag":136,"props":293,"children":294},{},[295],{"type":24,"value":296},"2.2 获得策略文件",{"type":18,"tag":193,"props":298,"children":300},{"code":299},"cd /path/to/your/mindformers/research/deepseek3\n\nbash scripts/msrun_launcher.sh \"run_mindformer.py \\\n--register_path research/deepseek3 \\ \n--config /path/to/your/desired/model/yaml \\\n--run_mode finetune \\\n--train_data /path/to/mindrecord \" 8 PORT output/msrun_log False 2000\n",[301],{"type":18,"tag":198,"props":302,"children":303},{"__ignoreMap":7},[304],{"type":24,"value":299},{"type":18,"tag":26,"props":306,"children":307},{},[308],{"type":24,"value":309},"参数说明：",{"type":18,"tag":193,"props":311,"children":313},{"code":312},"# run_mindformer.py 参数\nconfig: 模型的配置文件\nrun_mode: 运行模式选微调\ntrain_data: 训练用的数据文件\n# msrun_launcher.sh 参数\n单机上卡数8\nPORT为节点PORT\njoin=False\ntimeout=2000\n",[314],{"type":18,"tag":198,"props":315,"children":316},{"__ignoreMap":7},[317],{"type":24,"value":312},{"type":18,"tag":26,"props":319,"children":320},{},[321],{"type":24,"value":322},"生成的策略文件在strategy下，在下一步切分ckpt时作为dst_strategy的值。",{"type":18,"tag":26,"props":324,"children":325},{},[326],{"type":18,"tag":136,"props":327,"children":328},{},[329],{"type":24,"value":330},"2.3 获得特定切分的ckpt",{"type":18,"tag":193,"props":332,"children":334},{"code":333},"nohup python transform_checkpoint.py --src_checkpoint=/path/to/checkpoint.ckpt --dst_checkpoint=/path/to/desired/ckpt/ --dst_strategy=/path/to/strategy/ > output.log 2>&1 &\n",[335],{"type":18,"tag":198,"props":336,"children":337},{"__ignoreMap":7},[338],{"type":24,"value":333},{"type":18,"tag":26,"props":340,"children":341},{},[342],{"type":24,"value":206},{"type":18,"tag":193,"props":344,"children":346},{"code":345},"src_checkpoint：原始权重路径\ndst_checkpoint：目标权重路径\ndst_strategy：目标权重策略文件路径\n",[347],{"type":18,"tag":198,"props":348,"children":349},{"__ignoreMap":7},[350],{"type":24,"value":345},{"type":18,"tag":26,"props":352,"children":353},{},[354],{"type":18,"tag":136,"props":355,"children":356},{},[357],{"type":24,"value":358},"3. 训练/推理模型配置",{"type":18,"tag":26,"props":360,"children":361},{},[362],{"type":18,"tag":136,"props":363,"children":364},{},[365],{"type":24,"value":366},"3.1 训练模型配置",{"type":18,"tag":26,"props":368,"children":369},{},[370],{"type":24,"value":371},"训练的模型的配置finetune_deepseek3_671b. yaml:",{"type":18,"tag":193,"props":373,"children":375},{"code":374},"parallel_config:\n  data_parallel: 1 # 数据并行切分为 4\n  model_parallel: 4 # 模型并行切分为 1\n  pipeline_stage: 2 # 流水线并行切分为 1\n  expert_parallel: 1# 专家并行切分为 4\n  use_seq_parallel: True\n  micro_batch_num: 2\n  vocab_emb_dp: False\n  gradient_aggregation_group: 4\n  micro_batch_interleave_num: 2 # mp大于1时，设为1可提升训练效率\nmodel_config:\n  num_layers: &num_layers 3\n",[376],{"type":18,"tag":198,"props":377,"children":378},{"__ignoreMap":7},[379],{"type":24,"value":374},{"type":18,"tag":26,"props":381,"children":382},{},[383],{"type":24,"value":384},"在本教程中，将DeepSeek V3的层数裁剪为3层使得可以本地执行任务。",{"type":18,"tag":26,"props":386,"children":387},{},[388],{"type":24,"value":389},"训练相关配置在mindrlhf/configs/grpo_configs.py有学习率和GRPO相关的超参",{"type":18,"tag":193,"props":391,"children":393},{"code":392},"optimizer: str = 'adamw' # 优化器类型\nbeta1: float = 0.9  # 优化器adamw超参，下同\nbeta2: float = 0.95 \neps: float = 1.0e-8 \nweight_decay: float = 0.01\n\nepochs: int = 100 # 训练轮数\n",[394],{"type":18,"tag":198,"props":395,"children":396},{"__ignoreMap":7},[397],{"type":24,"value":392},{"type":18,"tag":26,"props":399,"children":400},{},[401],{"type":18,"tag":136,"props":402,"children":403},{},[404],{"type":24,"value":405},"3.2 推理模型配置",{"type":18,"tag":26,"props":407,"children":408},{},[409],{"type":24,"value":410},"推理的模型的配置predict_deepseek3_671b.yaml",{"type":18,"tag":193,"props":412,"children":414},{"code":413},"parallel_config:\n  data_parallel: 2\n  model_parallel: 4 # 模型并行切分为4\n  pipeline_stage: 1 # 流水线并行切分为1\n  expert_parallel: 1# 专家并行切分为1\n  micro_batch_num: 1\n  vocab_emb_dp: False\n  gradient_aggregation_group: 4\n  micro_batch_interleave_num: 1\nmodel_config:\n  num_layers: &num_layers 3\n",[415],{"type":18,"tag":198,"props":416,"children":417},{"__ignoreMap":7},[418],{"type":24,"value":413},{"type":18,"tag":26,"props":420,"children":421},{},[422],{"type":24,"value":384},{"type":18,"tag":26,"props":424,"children":425},{},[426],{"type":18,"tag":136,"props":427,"children":428},{},[429],{"type":24,"value":430},"三****启动单机8卡训练脚本",{"type":18,"tag":26,"props":432,"children":433},{},[434],{"type":24,"value":435},"用bash run_grpo.sh启动GRPO强化学习流程。",{"type":18,"tag":26,"props":437,"children":438},{},[439],{"type":24,"value":440},"注意：用户需要确认将MindSpore Transformers和MindSpore RLHF的路径加入PYTHONPATH。",{"type":18,"tag":193,"props":442,"children":444},{"code":443},"msrun --worker_num=8 --local_worker_num=8 --master_addr=127.0.0.1 \\\n--master_port=9190 --join=False --log_dir=./deepseekv3_one_log \\\nexamples/grpo/deepseekv3_grpo_tutorial/grpo_one_stage.py \\\n--sft_path_infer ./model_configs/deepseek_v3_config/predict_deepseek3_671b.yaml \\\n--sft_path_train ./model_configs/deepseek_v3_config/finetune_deepseek3_671b.yaml \\\n--tokenizer_path /path/to/tokenizer.json \\\n--mind_dataset_dir /path/to/gms8k.mindrecord \\\n--save_data_file /path/to/grpo.mindrecord \\\n--save_ckpt_dir /path/to/save/ckpt \\\n--use_parallel True \\\n--load_sft_checkpoint_infer /path/to/infer/ckpt \\\n--load_sft_checkpoint_train /path/to/train/ckpt \\\n--load_ref_checkpoint /path/to/ref/ckpt \\\n--enable_compile_cache False \\\n",[445],{"type":18,"tag":198,"props":446,"children":447},{"__ignoreMap":7},[448],{"type":24,"value":443},{"type":18,"tag":26,"props":450,"children":451},{},[452],{"type":24,"value":206},{"type":18,"tag":193,"props":454,"children":456},{"code":455},"# msrun 参数\nworker_num：总卡数\nlocal_worker_num：单机的卡数\nmaster_addr：主节点地址\nmaster_port: 主节点端口\njoin：是否等待所有worker退出\nlog_dir: 日志路径\n# grpo_one_stage.py 参数\nsft_path_infer：推理用的模型配置\nsft_path_train：训练用的模型配置\ntokenizer_path：tokenizer.json文件路径\nmind_dataset_dir：训练数据文件的路径\nsave_data_file：中间推理结果的保存路径（可选）\nsave_ckpt_dir：训练ckpt的保存路径\nuse_parallel：是否并行\nload_sft_checkpoint_infer: 推理ckpt路径\nload_sft_checkpoint_train: 训练ckpt路径\nload_ref_checkpoint：参考模型ckpt路径\nenable_compile_cache：是否编译缓存\n",[457],{"type":18,"tag":198,"props":458,"children":459},{"__ignoreMap":7},[460],{"type":24,"value":455},{"type":18,"tag":26,"props":462,"children":463},{},[464],{"type":24,"value":465},"拉起任务后，通过",{"type":18,"tag":193,"props":467,"children":469},{"code":468},"tail -f deepseekv3_one_log/worker_0.log\n",[470],{"type":18,"tag":198,"props":471,"children":472},{"__ignoreMap":7},[473],{"type":24,"value":468},{"title":7,"searchDepth":475,"depth":475,"links":476},4,[],"markdown","content:technology-blogs:zh:3651.md","content","technology-blogs/zh/3651.md","technology-blogs/zh/3651","md",1776506132932]