[{"data":1,"prerenderedAt":497},["ShallowReactive",2],{"content-query-O0hC8pO9L7":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":491,"_id":492,"_source":493,"_file":494,"_stem":495,"_extension":496},"/technology-blogs/zh/3661","zh",false,"","视频生成效果惊艳！昇思率先完成Open-Sora 2.0支持","Open-Sora 2.0是潞晨科技推出的全新开源SOTA视频生成模型，相比传统高性能视频生成模型大幅降低了训练成本。","2025-03-25","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/26/642cee8d2668420fa8a651bc45024dba.png","technology-blogs","实践",{"type":15,"children":16,"toc":477},"root",[17,25,31,44,63,80,88,100,107,122,130,153,161,184,189,196,201,211,216,231,239,246,278,286,294,302,310,318,326,331,340,345,350,359,364,369,377,385,390,398,406,411,419,428,436,441,448,457,465],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"视频生成效果惊艳昇思率先完成open-sora-20支持",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":28},"p",{},[29],{"type":24,"value":30},"Open-Sora 2.0是潞晨科技推出的全新开源SOTA视频生成模型，相比传统高性能视频生成模型大幅降低了训练成本。Open-Sora 2.0基于3D自编码器 、3D全注意力机制和MMDiT架构，结合高效的并行训练方案和高压缩比自编码器，显著提升训练效率和推理速度。",{"type":18,"tag":26,"props":32,"children":33},{},[34,36,42],{"type":24,"value":35},"MindSpore团队现已完成对Open-Sora 2.0的适配，并将其开源至",{"type":18,"tag":37,"props":38,"children":39},"strong",{},[40],{"type":24,"value":41},"MindSpore ONE",{"type":24,"value":43},"仓库，本文将要给大家详细介绍，如何基于昇思MindSpore和单机Atlas 800T A2，完整实现Open-Sora 2.0视频生成流程。",{"type":18,"tag":45,"props":46,"children":47},"ul",{},[48],{"type":18,"tag":49,"props":50,"children":51},"li",{},[52,54],{"type":24,"value":53},"MindSpore ONE开源代码仓链接：",{"type":18,"tag":55,"props":56,"children":60},"a",{"href":57,"rel":58},"https://github.com/mindspore-lab/mindone/tree/master/examples/opensora%5C_hpcai",[59],"nofollow",[61],{"type":24,"value":62},"https://github.com/mindspore-lab/mindone/tree/master/examples/opensora\\_hpcai",{"type":18,"tag":64,"props":65,"children":67},"h3",{"id":66},"_01-效果展示",[68,73,75],{"type":18,"tag":37,"props":69,"children":70},{},[71],{"type":24,"value":72},"# 01",{"type":24,"value":74}," ",{"type":18,"tag":37,"props":76,"children":77},{},[78],{"type":24,"value":79},"效果展示",{"type":18,"tag":26,"props":81,"children":82},{},[83],{"type":18,"tag":84,"props":85,"children":87},"img",{"alt":7,"src":86},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/28/6fca806b1a0f436f90a95caac6e1bf88.png",[],{"type":18,"tag":26,"props":89,"children":90},{},[91,95,96],{"type":18,"tag":84,"props":92,"children":94},{"alt":7,"src":93},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/28/0a614c0eca3d4e2bbe724b76b5c17c5e.png",[],{"type":24,"value":74},{"type":18,"tag":84,"props":97,"children":99},{"alt":7,"src":98},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/28/25e54b81ac074517a4d1286f3472ade5.png",[],{"type":18,"tag":26,"props":101,"children":102},{},[103],{"type":18,"tag":84,"props":104,"children":106},{"alt":7,"src":105},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/28/24b5d615af634497895891830ab5746d.png",[],{"type":18,"tag":64,"props":108,"children":110},{"id":109},"_02-模型介绍",[111,116,117],{"type":18,"tag":37,"props":112,"children":113},{},[114],{"type":24,"value":115},"# 02",{"type":24,"value":74},{"type":18,"tag":37,"props":118,"children":119},{},[120],{"type":24,"value":121},"模型介绍",{"type":18,"tag":26,"props":123,"children":124},{},[125],{"type":18,"tag":37,"props":126,"children":127},{},[128],{"type":24,"value":129},"1、Open-Sora 2.0 的主要功能",{"type":18,"tag":45,"props":131,"children":132},{},[133,138,143,148],{"type":18,"tag":49,"props":134,"children":135},{},[136],{"type":24,"value":137},"**高****质量视频生成：**生成 720p 分辨率、24 FPS 的流畅视频，支持多种场景和风格，从自然光到复杂动态场景都能表现出色。",{"type":18,"tag":49,"props":139,"children":140},{},[141],{"type":24,"value":142},"**动作幅度可控：**根据用户需求调整视频中人物或物体的动作幅度，实现更细腻、精准的动态表现。",{"type":18,"tag":49,"props":144,"children":145},{},[146],{"type":24,"value":147},"**文本到视频（T2V）生成：**支持用文本描述直接生成对应的视频内容，满足创意视频制作和内容生成的需求。",{"type":18,"tag":49,"props":149,"children":150},{},[151],{"type":24,"value":152},"**图像到视频（I2V）生成：**结合开源图像模型，基于图像生成视频，进一步提升生成效果和多样性。",{"type":18,"tag":26,"props":154,"children":155},{},[156],{"type":18,"tag":37,"props":157,"children":158},{},[159],{"type":24,"value":160},"2、Open-Sora 2.0 的技术原理",{"type":18,"tag":45,"props":162,"children":163},{},[164,169,174,179],{"type":18,"tag":49,"props":165,"children":166},{},[167],{"type":24,"value":168},"**模型架构：**基于三维自编码器高效处理视频数据，捕捉时间维度上的动态信息。引入全注意力机制，提升视频生成的时空一致性。结合多模态扩散（MMDIT）架构，更精准地捕捉文本与视频内容的关联。",{"type":18,"tag":49,"props":170,"children":171},{},[172],{"type":24,"value":173},"**高压缩比自编码器：**基于 4×32×32 的高压缩比自编码器，显著降低推理成本。",{"type":18,"tag":49,"props":175,"children":176},{},[177],{"type":24,"value":178},"**高效训练方法：**基于多阶段、多层次的数据筛选机制，确保高质量数据输入，提升训练效率。优先在低分辨率下训练，学习关键动态特征，逐步提升分辨率，大幅降低计算开销。优先训练图生视频任务，用图像特征引导视频生成，加速模型收敛。",{"type":18,"tag":49,"props":180,"children":181},{},[182],{"type":24,"value":183},"**模型初始化与蒸馏：**借助开源图生视频模型 FLUX 进行初始化，降低训练成本。基于蒸馏的优化策略提升自编码器特征空间的表达能力，减少训练所需数据量和时间。",{"type":18,"tag":26,"props":185,"children":186},{},[187],{"type":24,"value":188},"Open-Sora diffusion transformer架构图如下：",{"type":18,"tag":26,"props":190,"children":191},{},[192],{"type":18,"tag":84,"props":193,"children":195},{"alt":7,"src":194},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/28/44f960ae6e6c42289a2a6a7efe5dac26.png",[],{"type":18,"tag":26,"props":197,"children":198},{},[199],{"type":24,"value":200},"在代码实现上，关键的处理包括VAE模块对图片/视频的高比例压缩。Open-Sora 2.0借鉴了hunyuan_vae模块，核心压缩代码如下:",{"type":18,"tag":202,"props":203,"children":205},"pre",{"code":204},"def spatial_tiled_encode(self, x: ms.tensor, return_moments: bool = False) -> DiagonalGaussianDistribution:\n        r\"\"\"Encode a batch of images/videos using a tiled encoder.\n\n        ....\n            for j in range(0, x.shape[-1], overlap_size):\n                tile = x[:, :, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size]\n                tile = self.encoder(tile)    # hunyuan_vae对图片/视频进行高比例压缩\n                tile = self.quant_conv(tile)\n                row.append(tile)\n            ...\n        posterior = DiagonalGaussianDistribution(moments)\n        return posterior\n",[206],{"type":18,"tag":207,"props":208,"children":209},"code",{"__ignoreMap":7},[210],{"type":24,"value":204},{"type":18,"tag":26,"props":212,"children":213},{},[214],{"type":24,"value":215},"通过该高压缩比视频自编码器，视频生成成本将大幅降低，推理加速效果显著。",{"type":18,"tag":64,"props":217,"children":219},{"id":218},"_03-快速上手",[220,225,226],{"type":18,"tag":37,"props":221,"children":222},{},[223],{"type":24,"value":224},"# 03",{"type":24,"value":74},{"type":18,"tag":37,"props":227,"children":228},{},[229],{"type":24,"value":230},"快速上手",{"type":18,"tag":26,"props":232,"children":233},{},[234],{"type":18,"tag":37,"props":235,"children":236},{},[237],{"type":24,"value":238},"1、环境准备",{"type":18,"tag":26,"props":240,"children":241},{},[242],{"type":18,"tag":84,"props":243,"children":245},{"alt":7,"src":244},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/28/59dc7b0aa95549ab90f8b30fbab1c588.png",[],{"type":18,"tag":45,"props":247,"children":248},{},[249,264],{"type":18,"tag":49,"props":250,"children":251},{},[252,254,258],{"type":24,"value":253},"CANN下载：",{"type":18,"tag":255,"props":256,"children":257},"br",{},[],{"type":18,"tag":55,"props":259,"children":262},{"href":260,"rel":261},"https://www.hiascend.com/developer/download/community/result?cann=8.0.0.beta1",[59],[263],{"type":24,"value":260},{"type":18,"tag":49,"props":265,"children":266},{},[267,269,272],{"type":24,"value":268},"MindSpore下载：",{"type":18,"tag":255,"props":270,"children":271},{},[],{"type":18,"tag":55,"props":273,"children":276},{"href":274,"rel":275},"https://www.mindspore.cn/install",[59],[277],{"type":24,"value":274},{"type":18,"tag":26,"props":279,"children":280},{},[281],{"type":18,"tag":37,"props":282,"children":283},{},[284],{"type":24,"value":285},"2、安装依赖",{"type":18,"tag":202,"props":287,"children":289},{"code":288},"git clone https://github.com/mindspore-lab/mindone\ncd mindone/examples/opensora_hpcai\npip install -r requirements.txt\n",[290],{"type":18,"tag":207,"props":291,"children":292},{"__ignoreMap":7},[293],{"type":24,"value":288},{"type":18,"tag":26,"props":295,"children":296},{},[297],{"type":18,"tag":37,"props":298,"children":299},{},[300],{"type":24,"value":301},"3、模型下载",{"type":18,"tag":26,"props":303,"children":304},{},[305],{"type":18,"tag":37,"props":306,"children":307},{},[308],{"type":24,"value":309},"模型",{"type":18,"tag":26,"props":311,"children":312},{},[313],{"type":18,"tag":37,"props":314,"children":315},{},[316],{"type":24,"value":317},"下载链接",{"type":18,"tag":26,"props":319,"children":320},{},[321],{"type":18,"tag":37,"props":322,"children":323},{},[324],{"type":24,"value":325},"说明",{"type":18,"tag":26,"props":327,"children":328},{},[329],{"type":24,"value":330},"hpcai-tech/Open-Sora-v2",{"type":18,"tag":26,"props":332,"children":333},{},[334],{"type":18,"tag":55,"props":335,"children":338},{"href":336,"rel":337},"https://huggingface.co/hpcai-tech/Open-Sora-v2",[59],[339],{"type":24,"value":336},{"type":18,"tag":26,"props":341,"children":342},{},[343],{"type":24,"value":344},"Open-Sora 2.0权重，包括模型核心权重、VAE权重和text encoder权重",{"type":18,"tag":26,"props":346,"children":347},{},[348],{"type":24,"value":349},"hpcai-tech/Open-Sora-v2-Video-DC-AE",{"type":18,"tag":26,"props":351,"children":352},{},[353],{"type":18,"tag":55,"props":354,"children":357},{"href":355,"rel":356},"https://huggingface.co/hpcai-tech/Open-Sora-v2-Video-DC-AE",[59],[358],{"type":24,"value":355},{"type":18,"tag":26,"props":360,"children":361},{},[362],{"type":24,"value":363},"Open-Sora 2.0 DC-AE编码器权重",{"type":18,"tag":26,"props":365,"children":366},{},[367],{"type":24,"value":368},"从 Hugging Face 下载所需的模型，可以参考如下命令：",{"type":18,"tag":202,"props":370,"children":372},{"code":371},"huggingface-cli download hpcai-tech/Open-Sora-v2 --local-dir ./ckpts\n",[373],{"type":18,"tag":207,"props":374,"children":375},{"__ignoreMap":7},[376],{"type":24,"value":371},{"type":18,"tag":26,"props":378,"children":379},{},[380],{"type":18,"tag":37,"props":381,"children":382},{},[383],{"type":24,"value":384},"4、运行推理",{"type":18,"tag":26,"props":386,"children":387},{},[388],{"type":24,"value":389},"生成text embedding，包括真实的prompts和negative prompts。",{"type":18,"tag":202,"props":391,"children":393},{"code":392},"TRANSFORMERS_OFFLINE=1 python scripts/v2.0/text_embedding.py --model.from_pretrained=openai/clip-vit-large-patch14 --model.max_length=77 --prompts_file=FILE_WITH_PROMPTS --output_path=assets/texts/clip\n",[394],{"type":18,"tag":207,"props":395,"children":396},{"__ignoreMap":7},[397],{"type":24,"value":392},{"type":18,"tag":202,"props":399,"children":401},{"code":400},"\nTRANSFORMERS_OFFLINE=1 python scripts/v2.0/text_embedding.py --model.from_pretrained=DeepFloyd/t5-v1_1-xxl --model.max_length=512 --prompts_file=FILE_WITH_PROMPTS --output_path=assets/texts/t5_512\n",[402],{"type":18,"tag":207,"props":403,"children":404},{"__ignoreMap":7},[405],{"type":24,"value":400},{"type":18,"tag":26,"props":407,"children":408},{},[409],{"type":24,"value":410},"开始推理",{"type":18,"tag":202,"props":412,"children":414},{"code":413},"\npython scripts/v2.0/inference_v2.py \n--config=configs/opensora-v2-0/inference/256px.yaml \n--text_emb.t5_dir=FOLDER_WITH_T5_PROMPTS \n--text_emb.neg_t5_dir=FOLDER_WITH_T5_NEG_PROMPTS \n--text_emb.clip_dir=FOLDER_WITH_CLIP_PROMPTS \n--text_emb.neg_clip_dir=FOLDER_WITH_CLIP_NEG_PROMPTS\n",[415],{"type":18,"tag":207,"props":416,"children":417},{"__ignoreMap":7},[418],{"type":24,"value":413},{"type":18,"tag":64,"props":420,"children":422},{"id":421},"_04",[423],{"type":18,"tag":37,"props":424,"children":425},{},[426],{"type":24,"value":427},"# 04",{"type":18,"tag":64,"props":429,"children":431},{"id":430},"性能实测",[432],{"type":18,"tag":37,"props":433,"children":434},{},[435],{"type":24,"value":430},{"type":18,"tag":26,"props":437,"children":438},{},[439],{"type":24,"value":440},"基于Atlas 800T A2和MindSpore2.5.0的性能测试结果如下：",{"type":18,"tag":26,"props":442,"children":443},{},[444],{"type":18,"tag":84,"props":445,"children":447},{"alt":7,"src":446},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/03/28/c4e12ef700a940269dc4dca7fa89b67a.png",[],{"type":18,"tag":64,"props":449,"children":451},{"id":450},"_05",[452],{"type":18,"tag":37,"props":453,"children":454},{},[455],{"type":24,"value":456},"# 05",{"type":18,"tag":64,"props":458,"children":460},{"id":459},"马上体验",[461],{"type":18,"tag":37,"props":462,"children":463},{},[464],{"type":24,"value":459},{"type":18,"tag":26,"props":466,"children":467},{},[468,470],{"type":24,"value":469},"魔乐社区也已完成Open-Sora 2.0，欢迎体验：",{"type":18,"tag":55,"props":471,"children":474},{"href":472,"rel":473},"https://modelers.cn/spaces/MindSpore-Lab/Sora%5C_hpcai%5C_2.0",[59],[475],{"type":24,"value":476},"https://modelers.cn/spaces/MindSpore-Lab/Sora\\_hpcai\\_2.0",{"title":7,"searchDepth":478,"depth":478,"links":479},4,[480,483,485,487,488,489,490],{"id":66,"depth":481,"text":482},3,"# 01 效果展示",{"id":109,"depth":481,"text":484},"# 02 模型介绍",{"id":218,"depth":481,"text":486},"# 03 快速上手",{"id":421,"depth":481,"text":427},{"id":430,"depth":481,"text":430},{"id":450,"depth":481,"text":456},{"id":459,"depth":481,"text":459},"markdown","content:technology-blogs:zh:3661.md","content","technology-blogs/zh/3661.md","technology-blogs/zh/3661","md",1776506133032]