[{"data":1,"prerenderedAt":423},["ShallowReactive",2],{"content-query-yS0cyG7phi":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":417,"_id":418,"_source":419,"_file":420,"_stem":421,"_extension":422},"/technology-blogs/zh/3752","zh",false,"","扩散模型系列——LDM","LDM   论文地址：High-Resolution Image Synthesis with Latent Diffusion Models    代码地址：https://github.com/CompVis/latent-diffusion","2025-05-07","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/06/06/ffdad334290c44fcb1f8e48745183662.png","technology-blogs","开发者说",{"type":15,"children":16,"toc":391},"root",[17,25,32,47,58,65,70,75,100,106,113,118,124,129,135,140,146,152,157,171,177,182,195,201,206,219,225,231,244,250,263,269,337,343,356,362,367,375,381],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"扩散模型系列ldm",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":29},"h2",{"id":28},"ldm",[30],{"type":24,"value":31},"LDM",{"type":18,"tag":33,"props":34,"children":35},"p",{},[36,38],{"type":24,"value":37},"论文地址：",{"type":18,"tag":39,"props":40,"children":44},"a",{"href":41,"rel":42},"https://arxiv.org/pdf/2112.10752",[43],"nofollow",[45],{"type":24,"value":46},"High-Resolution Image Synthesis with Latent Diffusion Models",{"type":18,"tag":33,"props":48,"children":49},{},[50,52],{"type":24,"value":51},"代码地址：",{"type":18,"tag":39,"props":53,"children":56},{"href":54,"rel":55},"https://github.com/CompVis/latent-diffusion",[43],[57],{"type":24,"value":54},{"type":18,"tag":59,"props":60,"children":62},"h3",{"id":61},"一概述",[63],{"type":24,"value":64},"一、概述",{"type":18,"tag":33,"props":66,"children":67},{},[68],{"type":24,"value":69},"LDM（Latent Diffusion Model）是一种基于扩散过程的先进生成模型，其核心思想是通过逐步去除图像中的噪声来生成高质量的图像样本。与传统扩散模型不同，LDM 引入了潜在空间的概念，将图像表示为潜在空间中的向量。通过学习潜在空间中的映射关系，结合预训练好的变分自编码器（VAE），实现高效的图像生成。由于 LDM 在潜在空间进行操作，特征维度远小于图像空间，因此推理速度相较于基于 DDPM 的模型有显著提升。",{"type":18,"tag":33,"props":71,"children":72},{},[73],{"type":24,"value":74},"LDM 的主要特点包括：",{"type":18,"tag":76,"props":77,"children":78},"ol",{},[79,85,90,95],{"type":18,"tag":80,"props":81,"children":82},"li",{},[83],{"type":24,"value":84},"VAE 潜在表示学习：利用 VAE 学习图像的潜在表示，在压缩数据的同时保留关键特征，为后续的扩散过程提供低维高效的输入。",{"type":18,"tag":80,"props":86,"children":87},{},[88],{"type":24,"value":89},"UNet 映射学习：采用 UNet 架构学习从潜在空间到图像空间的映射关系，充分利用 UNet 的多尺度特征提取和重建能力。",{"type":18,"tag":80,"props":91,"children":92},{},[93],{"type":24,"value":94},"条件扩散生成：支持条件扩散过程，能够根据给定的条件（如文本、图像等）生成特定类型的图像，增强了模型的可控性。",{"type":18,"tag":80,"props":96,"children":97},{},[98],{"type":24,"value":99},"分层扩散机制：运用分层扩散过程，在不同的分辨率层级上进行图像生成，有助于生成高分辨率、细节丰富的图像。",{"type":18,"tag":59,"props":101,"children":103},{"id":102},"二主要步骤",[104],{"type":24,"value":105},"二、主要步骤",{"type":18,"tag":107,"props":108,"children":110},"h4",{"id":109},"_1-数据编码",[111],{"type":24,"value":112},"1. 数据编码",{"type":18,"tag":33,"props":114,"children":115},{},[116],{"type":24,"value":117},"使用预训练的 VAE 编码器将原始高维图像数据映射到低维潜在空间。这一过程不仅降低了计算复杂度，还提取了图像的关键特征，为后续的扩散过程提供了更紧凑的表示。",{"type":18,"tag":107,"props":119,"children":121},{"id":120},"_2-潜在空间扩散",[122],{"type":24,"value":123},"2. 潜在空间扩散",{"type":18,"tag":33,"props":125,"children":126},{},[127],{"type":24,"value":128},"在潜在空间中执行与 DDPM 类似的扩散过程。通过逐步向潜在表示添加高斯噪声，将其从原始数据分布转化为纯噪声分布。在反向过程中，通过 UNet 逐步去除噪声，生成新的潜在样本。",{"type":18,"tag":107,"props":130,"children":132},{"id":131},"_3-数据重建",[133],{"type":24,"value":134},"3. 数据重建",{"type":18,"tag":33,"props":136,"children":137},{},[138],{"type":24,"value":139},"使用预训练好的 VAE 解码器将去噪后的潜在表示映射回原始图像空间，得到最终的生成图像。",{"type":18,"tag":59,"props":141,"children":143},{"id":142},"三关键点",[144],{"type":24,"value":145},"三、关键点",{"type":18,"tag":107,"props":147,"children":149},{"id":148},"_1-感知压缩与潜在空间优化",[150],{"type":24,"value":151},"1. 感知压缩与潜在空间优化",{"type":18,"tag":33,"props":153,"children":154},{},[155],{"type":24,"value":156},"LDM 通过预训练的自动编码器将高维图像数据进行感知压缩，映射到低维潜在空间。例如，将 256×256 的图像压缩为 16×16 的潜在表示，大幅减少了计算量。为了控制潜在空间的分布，LDM 采用了以下两种正则化技术：",{"type":18,"tag":158,"props":159,"children":160},"ul",{},[161,166],{"type":18,"tag":80,"props":162,"children":163},{},[164],{"type":24,"value":165},"KL 正则化：约束潜在变量接近标准正态分布，类似于 VAE 的做法，增强了生成过程的稳定性。",{"type":18,"tag":80,"props":167,"children":168},{},[169],{"type":24,"value":170},"VQ 正则化：通过向量量化层对潜在表示进行离散化，类似于 VQ - VAE，有助于提升模型对结构化特征的学习能力。",{"type":18,"tag":107,"props":172,"children":174},{"id":173},"_2-扩散过程与去噪机制",[175],{"type":24,"value":176},"2. 扩散过程与去噪机制",{"type":18,"tag":33,"props":178,"children":179},{},[180],{"type":24,"value":181},"LDM 在潜在空间中进行扩散过程，分为前向扩散和反向去噪两个阶段：",{"type":18,"tag":158,"props":183,"children":184},{},[185,190],{"type":18,"tag":80,"props":186,"children":187},{},[188],{"type":24,"value":189},"前向扩散：逐步向潜在变量添加高斯噪声，噪声强度随时间步递增（如采用线性或余弦调度），最终使潜在表示接近纯噪声分布。",{"type":18,"tag":80,"props":191,"children":192},{},[193],{"type":24,"value":194},"反向去噪：使用 UNet 预测当前潜在表示中的噪声，并逐步去除噪声。目标函数为预测噪声与实际噪声的均方误差，通过优化该目标函数提升模型的去噪能力。",{"type":18,"tag":107,"props":196,"children":198},{"id":197},"_3-多模态条件生成",[199],{"type":24,"value":200},"3. 多模态条件生成",{"type":18,"tag":33,"props":202,"children":203},{},[204],{"type":24,"value":205},"LDM 引入了交叉注意力机制，支持文本、图像、语义地图等多模态条件输入。具体实现方式如下：",{"type":18,"tag":158,"props":207,"children":208},{},[209,214],{"type":18,"tag":80,"props":210,"children":211},{},[212],{"type":24,"value":213},"条件编码器：将不同模态的输入（如文本、图像）映射为中间表示，再与 UNet 的中间层进行交互。",{"type":18,"tag":80,"props":215,"children":216},{},[217],{"type":24,"value":218},"注意力融合：通过查询（Q）、键（K）、值（V）机制，将条件信息融入生成过程，实现对图像生成的精准控制，例如根据文本描述生成相应的图像。",{"type":18,"tag":59,"props":220,"children":222},{"id":221},"四模型结构",[223],{"type":24,"value":224},"四、模型结构",{"type":18,"tag":107,"props":226,"children":228},{"id":227},"_1-像素空间与潜在空间的转换",[229],{"type":24,"value":230},"1. 像素空间与潜在空间的转换",{"type":18,"tag":158,"props":232,"children":233},{},[234,239],{"type":18,"tag":80,"props":235,"children":236},{},[237],{"type":24,"value":238},"编码器（Encoder）：由卷积网络构成，通过下采样和特征提取操作，将输入图像 x从高维像素空间压缩到低维潜在空间，生成潜在表示 z。",{"type":18,"tag":80,"props":240,"children":241},{},[242],{"type":24,"value":243},"解码器（Decoder）：包含上采样和特征重建层，将去噪后的潜在表示 \\tilde{z}映射回像素空间，生成重建图像 \\tilde{x}。",{"type":18,"tag":107,"props":245,"children":247},{"id":246},"_2-潜在空间中的扩散过程",[248],{"type":24,"value":249},"2. 潜在空间中的扩散过程",{"type":18,"tag":158,"props":251,"children":252},{},[253,258],{"type":18,"tag":80,"props":254,"children":255},{},[256],{"type":24,"value":257},"前向扩散：在潜在空间中，逐步向潜在表示 z添加高斯噪声，生成一系列噪声版本 z_t（t表示时间步）。",{"type":18,"tag":80,"props":259,"children":260},{},[261],{"type":24,"value":262},"噪声调度：噪声强度随时间步递增，常见的调度方式有线性调度和余弦调度，最终使 z_T接近纯噪声。",{"type":18,"tag":107,"props":264,"children":266},{"id":265},"_3-去噪-u-net-核心模块",[267],{"type":24,"value":268},"3. 去噪 U-Net 核心模块",{"type":18,"tag":158,"props":270,"children":271},{},[272,295,313],{"type":18,"tag":80,"props":273,"children":274},{},[275,277],{"type":24,"value":276},"网络结构：\n",{"type":18,"tag":158,"props":278,"children":279},{},[280,285,290],{"type":18,"tag":80,"props":281,"children":282},{},[283],{"type":24,"value":284},"下采样路径：通过卷积层、批归一化（BN）和激活函数（如 ReLU）逐步提取多尺度特征。",{"type":18,"tag":80,"props":286,"children":287},{},[288],{"type":24,"value":289},"上采样路径：使用反卷积或上采样操作恢复空间分辨率。",{"type":18,"tag":80,"props":291,"children":292},{},[293],{"type":24,"value":294},"跳跃连接：将下采样层的特征直接传递到对应的上采样层，帮助保留图像的细节信息。",{"type":18,"tag":80,"props":296,"children":297},{},[298,300],{"type":24,"value":299},"时间条件注入：\n",{"type":18,"tag":158,"props":301,"children":302},{},[303,308],{"type":18,"tag":80,"props":304,"children":305},{},[306],{"type":24,"value":307},"通过可学习的时间嵌入（如正弦位置编码）将时间步 t编码为向量。",{"type":18,"tag":80,"props":309,"children":310},{},[311],{"type":24,"value":312},"将时间嵌入向量与 UNet 中间特征进行拼接，指导模型根据扩散进度调整去噪策略。",{"type":18,"tag":80,"props":314,"children":315},{},[316,318],{"type":24,"value":317},"跨注意力机制：\n",{"type":18,"tag":158,"props":319,"children":320},{},[321,326],{"type":18,"tag":80,"props":322,"children":323},{},[324],{"type":24,"value":325},"条件融合：将条件信息（如文本、语义图）通过条件模块（如 Transformer）编码为查询（Q）。",{"type":18,"tag":80,"props":327,"children":328},{},[329,331,335],{"type":24,"value":330},"潜在表示交互：将潜在表示 z_t作为键（K）和值（V），通过注意力机制与条件查询交互。公式：",{"type":18,"tag":332,"props":333,"children":334},"br",{},[],{"type":24,"value":336},"\\text{Attention}(Q, K, V) = \\text{softmax}\\left(\\frac{QK^T}{\\sqrt{d}}\\right) \\cdot V",{"type":18,"tag":107,"props":338,"children":340},{"id":339},"_4-条件输入处理",[341],{"type":24,"value":342},"4. 条件输入处理",{"type":18,"tag":158,"props":344,"children":345},{},[346,351],{"type":18,"tag":80,"props":347,"children":348},{},[349],{"type":24,"value":350},"条件编码器：针对不同模态（如文本、图像）设计专用编码器，将输入映射为固定维度的向量。",{"type":18,"tag":80,"props":352,"children":353},{},[354],{"type":24,"value":355},"与 UNet 交互：条件编码器的输出作为跨注意力层的查询，指导 UNet 的去噪过程，从而实现根据条件生成图像。",{"type":18,"tag":107,"props":357,"children":359},{"id":358},"_5-去噪迭代过程",[360],{"type":24,"value":361},"5. 去噪迭代过程",{"type":18,"tag":33,"props":363,"children":364},{},[365],{"type":24,"value":366},"从纯噪声 z_T开始，通过 UNet 逐步去噪，每次迭代中 UNet 预测当前噪声并去除，最终生成干净的潜在表示 z_0，逐步逼近真实数据分布。",{"type":18,"tag":33,"props":368,"children":369},{},[370],{"type":18,"tag":371,"props":372,"children":374},"img",{"alt":7,"src":373},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2025/06/06/f650ea0b4a2d469386e942e23425f14f.png",[],{"type":18,"tag":59,"props":376,"children":378},{"id":377},"五代码实现",[379],{"type":24,"value":380},"五、代码实现",{"type":18,"tag":382,"props":383,"children":385},"pre",{"code":384},"import mindspore as ms\nfrom mindspore import nn, ops, Tensor, Parameter\nimport mindspore.numpy as mnp\nimport numpy as np\n\n# 超参数配置\nconfig = {\n    \"latent_dim\": 64,          # 潜在空间维度\n    \"image_size\": 64,          # 输入图像尺寸\n    \"batch_size\": 32,          # 批次大小\n    \"timesteps\": 1000,         # 扩散时间步数\n    \"lr\": 1e-4,                # 学习率\n    \"channels\": [64, 128, 256],# UNet通道数\n    \"condition_dim\": 128       # 条件嵌入维度\n}\n\n# 2. 编码器 - 解码器模块\nclass Encoder(nn.Cell):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(3, 32, 3, pad_mode='same', padding=1)\n        self.bn1 = nn.BatchNorm2d(32)\n        self.conv2 = nn.Conv2d(32, 64, 3, pad_mode='same', padding=1)\n        self.bn2 = nn.BatchNorm2d(64)\n        self.pool = nn.MaxPool2d(2, 2)\n        # 对于 64x64 输入，特征图尺寸为 64x16x16\n        self.fc = nn.Dense(64 * 16 * 16, config[\"latent_dim\"])\n\n    def construct(self, x):\n        x = ops.relu(self.bn1(self.conv1(x)))\n        x = self.pool(ops.relu(self.bn2(self.conv2(x))))\n        x = x.view(x.shape[0], -1)\n        return self.fc(x)\n\nclass Decoder(nn.Cell):\n    def __init__(self):\n        super().__init__()\n        self.fc = nn.Dense(config[\"latent_dim\"], 64 * 16 * 16)\n        self.conv1 = nn.Conv2d(64, 32, 3, pad_mode='same', padding=1)\n        self.bn1 = nn.BatchNorm2d(32)\n        self.conv2 = nn.Conv2d(32, 3, 3, pad_mode='same', padding=1)\n        self.upsample = nn.ResizeNearestNeighbor((32, 32))\n\n    def construct(self, z):\n        x = self.fc(z).view(-1, 64, 16, 16)\n        x = self.upsample(ops.relu(self.bn1(self.conv1(x))))\n        return self.conv2(x)\n\n# 3. 时间编码模块\nclass TimeEmbedding(nn.Cell):\n    def __init__(self, dim):\n        super().__init__()\n        self.embed = nn.Embedding(config[\"timesteps\"], dim)\n        self.proj = nn.Dense(dim, dim)\n\n    def construct(self, t):\n        return self.proj(self.embed(t))\n\n# 4. 交叉注意力模块\nclass CrossAttention(nn.Cell):\n    def __init__(self):\n        super().__init__()\n        self.q_proj = nn.Dense(config[\"latent_dim\"], config[\"condition_dim\"])\n        self.k_proj = nn.Dense(config[\"condition_dim\"], config[\"condition_dim\"])\n        self.v_proj = nn.Dense(config[\"condition_dim\"], config[\"condition_dim\"])\n        self.scale = mnp.sqrt(mnp.asarray(config[\"condition_dim\"], dtype=ms.float32))\n\n    def construct(self, z, condition):\n        q = self.q_proj(z)\n        k = self.k_proj(condition)\n        v = self.v_proj(condition)\n\n        attn = ops.matmul(q, k.transpose(0, 1)) / self.scale\n        attn = ops.softmax(attn, axis=-1)\n        return ops.matmul(attn, v)\n\n# 5. UNet 去噪网络\nclass UNetBlock(nn.Cell):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_channels, out_channels, 3, pad_mode='same', padding=1)\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.conv2 = nn.Conv2d(out_channels, out_channels, 3, pad_mode='same', padding=1)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n\n    def construct(self, x):\n        residual = x\n        x = ops.relu(self.bn1(self.conv1(x)))\n        x = self.bn2(self.conv2(x))\n        return x + residual  # 残差连接\n\nclass UNet(nn.Cell):\n    def __init__(self):\n        super().__init__()\n        self.time_emb = TimeEmbedding(config[\"latent_dim\"])\n        self.down1 = UNetBlock(3, config[\"channels\"][0])\n        self.down2 = UNetBlock(config[\"channels\"][0], config[\"channels\"][1])\n        self.down3 = UNetBlock(config[\"channels\"][1], config[\"channels\"][2])\n        self.up2 = UNetBlock(config[\"channels\"][1] + config[\"channels\"][2], config[\"channels\"][1])\n        self.up1 = UNetBlock(config[\"channels\"][0] + config[\"channels\"][1], config[\"channels\"][0])\n        self.final = nn.Conv2d(config[\"channels\"][0], 3, 3, pad_mode='same', padding=1)\n\n    def construct(self, x, t, condition):\n        # 时间条件注入\n        t_emb = self.time_emb(t)\n        t_emb = ops.broadcast_to(t_emb[:, None, None, :], (x.shape[0], x.shape[1], x.shape[2], t_emb.shape[1]))\n        x = ops.concat((x, t_emb), axis=-1)\n\n        # 下采样路径\n        x1 = self.down1(x)\n        x2 = self.down2(ops.max_pool2d(x1, 2))\n        x3 = self.down3(ops.max_pool2d(x2, 2))\n\n        # 上采样路径\n        x = self.up2(ops.concat((ops.resize_nearest_neighbor(x3, scale=2), x2), axis=1))\n        x = self.up1(ops.concat((ops.resize_nearest_neighbor(x, scale=2), x1), axis=1))\n\n        # 交叉注意力融合条件\n        attn = CrossAttention()(x.view(x.shape[0], -1, x.shape[-1]), condition)\n        x = x + attn.view(x.shape)\n\n        return self.final(x)\n\n# 6. 完整 LDM 模型\nclass LDM(nn.Cell):\n    def __init__(self):\n        super().__init__()\n        self.encoder = Encoder()\n        self.decoder = Decoder()\n        self.unet = UNet()\n        self.cross_attn = CrossAttention()\n\n    def construct(self, x, t, condition):\n        # 1. 编码到潜在空间\n        z = self.encoder(x)\n\n        # 2. 添加噪声（使用线性噪声调度）\n        beta = 1e-4 + (0.02 - 1e-4) * (t / config[\"timesteps\"])\n        noise = ops.randn(z.shape) * mnp.sqrt(beta)\n        z_noisy = z + noise\n\n        # 3. UNet 去噪\n        # 调整形状以适应 UNet 输入\n        denoised = self.unet(z_noisy.view(-1, 64, 8, 8), t, condition)\n\n        # 4. 解码回图像空间\n        return self.decoder(denoised)\n",[386],{"type":18,"tag":387,"props":388,"children":389},"code",{"__ignoreMap":7},[390],{"type":24,"value":384},{"title":7,"searchDepth":392,"depth":392,"links":393},4,[394],{"id":28,"depth":395,"text":31,"children":396},2,[397,399,404,409,416],{"id":61,"depth":398,"text":64},3,{"id":102,"depth":398,"text":105,"children":400},[401,402,403],{"id":109,"depth":392,"text":112},{"id":120,"depth":392,"text":123},{"id":131,"depth":392,"text":134},{"id":142,"depth":398,"text":145,"children":405},[406,407,408],{"id":148,"depth":392,"text":151},{"id":173,"depth":392,"text":176},{"id":197,"depth":392,"text":200},{"id":221,"depth":398,"text":224,"children":410},[411,412,413,414,415],{"id":227,"depth":392,"text":230},{"id":246,"depth":392,"text":249},{"id":265,"depth":392,"text":268},{"id":339,"depth":392,"text":342},{"id":358,"depth":392,"text":361},{"id":377,"depth":398,"text":380},"markdown","content:technology-blogs:zh:3752.md","content","technology-blogs/zh/3752.md","technology-blogs/zh/3752","md",1776506134556]