[{"data":1,"prerenderedAt":539},["ShallowReactive",2],{"content-query-w0PTRcTr7h":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":533,"_id":534,"_source":535,"_file":536,"_stem":537,"_extension":538},"/technology-blogs/zh/3293","zh",false,"","基于ms香橙派AIpro实现垃圾回收AI识别方案三：昇思大模型平台jupyter快速入门体验","四、昇思大模型平台jupyter快速入门体验：","2024-08-04","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/11/28/453c80492bf548d088ae4b771c8d3508.png","technology-blogs","调试调优",{"type":15,"children":16,"toc":521},"root",[17,25,38,48,57,67,77,87,91,95,104,109,117,122,129,134,141,146,156,162,167,172,180,187,192,200,206,211,219,224,232,237,245,250,258,263,271,278,284,289,297,304,310,315,335,340,353,358,366,371,379,384,391,397,402,410,416,421,434,442,447,452,460],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"基于ms香橙派aipro实现垃圾回收ai识别方案三昇思大模型平台jupyter快速入门体验",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":28},"p",{},[29],{"type":18,"tag":30,"props":31,"children":35},"a",{"href":32,"rel":33},"https://www.hiascend.com/developer/blog/details/0272157975874351541",[34],"nofollow",[36],{"type":24,"value":37},"基于ms香橙派AIpro实现垃圾回收AI识别方案一：昇思MindSpore介绍",{"type":18,"tag":26,"props":39,"children":40},{},[41],{"type":18,"tag":30,"props":42,"children":45},{"href":43,"rel":44},"https://www.hiascend.com/developer/blog/details/02111171533919729183",[34],[46],{"type":24,"value":47},"基于ms香橙派AIpro实现垃圾回收AI识别方案二：昇思MindSpore如何使用",{"type":18,"tag":26,"props":49,"children":50},{},[51],{"type":18,"tag":30,"props":52,"children":55},{"href":53,"rel":54},"https://www.hiascend.com/developer/blog/details/0272158138234394548",[34],[56],{"type":24,"value":8},{"type":18,"tag":26,"props":58,"children":59},{},[60],{"type":18,"tag":30,"props":61,"children":64},{"href":62,"rel":63},"https://www.hiascend.com/developer/blog/details/0272158143705280549",[34],[65],{"type":24,"value":66},"基于ms香橙派AIpro实现垃圾回收AI识别方案四：MindSpore应用实践 - 基于MobileNetv2的垃圾分类",{"type":18,"tag":26,"props":68,"children":69},{},[70],{"type":18,"tag":30,"props":71,"children":74},{"href":72,"rel":73},"https://www.hiascend.com/developer/blog/details/0272158155371368550",[34],[75],{"type":24,"value":76},"基于ms香橙派AIpro实现垃圾回收AI识别方案五：香橙派OrangePi AIpro实践(基于MobileNetv2的垃圾分类)",{"type":18,"tag":26,"props":78,"children":79},{},[80],{"type":18,"tag":30,"props":81,"children":84},{"href":82,"rel":83},"https://www.hiascend.com/developer/blog/details/0265158167508388542",[34],[85],{"type":24,"value":86},"基于ms香橙派AIpro实现垃圾回收AI识别方案六：SOWT分析与总结",{"type":18,"tag":88,"props":89,"children":90},"hr",{},[],{"type":18,"tag":26,"props":92,"children":93},{},[94],{"type":24,"value":9},{"type":18,"tag":96,"props":97,"children":98},"blockquote",{},[99],{"type":18,"tag":26,"props":100,"children":101},{},[102],{"type":24,"value":103},"本节通过MindSpore的API来快速实现一个简单的深度学习模型，若想要深入了解MindSpore的使用方法，后面还有10多个实验可以自行体验一下。",{"type":18,"tag":26,"props":105,"children":106},{},[107],{"type":24,"value":108},"上文提到了“AI实验室”中提供了“初学入门”、“应用实践”2种Demo，作为初学者可以先尝试体验一下“初学入门 – 初学教程 – 快速入门”的课程，如下左图一找到并双击打开初学教程，即可看到有官方提供的文档，从“文件 – 新建启动页”可以找开Jupyter的主要工作台。",{"type":18,"tag":26,"props":110,"children":111},{},[112],{"type":18,"tag":113,"props":114,"children":116},"img",{"alt":7,"src":115},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/08/31/682411d8f14d4592bc9ab0a27f5ed62d.png",[],{"type":18,"tag":26,"props":118,"children":119},{},[120],{"type":24,"value":121},"选择第一个python笔记本，将上面文档中的Cell代码直接进行复制，再放到Jupyter Lab中粘贴即可，不过，需要点击一下上面的“三角形”表示运行Cell项目。",{"type":18,"tag":123,"props":124,"children":126},"h2",{"id":125},"_41-安装相关依赖",[127],{"type":24,"value":128},"4.1 安装相关依赖：",{"type":18,"tag":26,"props":130,"children":131},{},[132],{"type":24,"value":133},"实验环境中已经预装了mindspore 2.3.0版本的，但是为了保险起见，还是建议先卸载mindspore，然后配置MINDSPORE_VERSION版本（如果想换成其它的版本，也可以修改），再进行安装，这里使用了华为的加速云地址下载，安装后，可以使用“!pip show mindspore”来查看版本。",{"type":18,"tag":26,"props":135,"children":136},{},[137],{"type":18,"tag":113,"props":138,"children":140},{"alt":7,"src":139},"https://fileserver.developer.huaweicloud.com/FileServer/getFile/cmtybbs/6a4/0ed/bc6/0421bb776e6a40edbc68b6389cfec54d.20240804070758.85403623442600406758205950764411:20240831021524:2400:DFE3CEBB430058847AECF4793AA3217B71E6DF5907B0E333F77B7C0727F982F2.jpg",[],{"type":18,"tag":26,"props":142,"children":143},{},[144],{"type":24,"value":145},"引入相关的包：",{"type":18,"tag":147,"props":148,"children":150},"pre",{"code":149},"import mindspore\nfrom mindspore import nn\nfrom mindspore.dataset import vision, transforms\nfrom mindspore.dataset import MnistDataset\n",[151],{"type":18,"tag":152,"props":153,"children":154},"code",{"__ignoreMap":7},[155],{"type":24,"value":149},{"type":18,"tag":123,"props":157,"children":159},{"id":158},"_42-处理数据集",[160],{"type":24,"value":161},"4.2 处理数据集：",{"type":18,"tag":26,"props":163,"children":164},{},[165],{"type":24,"value":166},"MindSpore提供基于Pipeline的数据引擎，通过数据集（Dataset）和数据变换（Transforms）实现高效的数据预处理。使用Mnist数据集，自动下载完成后，使用mindspore.dataset提供的数据变换进行预处理。",{"type":18,"tag":26,"props":168,"children":169},{},[170],{"type":24,"value":171},"使用download来下载文件并进行unzip解压缩，如果没有可以使用命令pip install download安装。",{"type":18,"tag":147,"props":173,"children":175},{"code":174},"# Download data from open datasets\nfrom download import download\n\nurl = \"https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/\" \\\n      \"notebook/datasets/MNIST_Data.zip\"\npath = download(url, \"./\", kind=\"zip\", replace=True)\n",[176],{"type":18,"tag":152,"props":177,"children":178},{"__ignoreMap":7},[179],{"type":24,"value":174},{"type":18,"tag":26,"props":181,"children":182},{},[183],{"type":18,"tag":113,"props":184,"children":186},{"alt":7,"src":185},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/08/31/035281ea132643a8a8aceabb014bcb73.png",[],{"type":18,"tag":26,"props":188,"children":189},{},[190],{"type":24,"value":191},"MNIST数据集目录结构如下：",{"type":18,"tag":147,"props":193,"children":195},{"code":194},"MNIST_Data\n└── train\n    ├── train-images-idx3-ubyte (60000个训练图片)\n    ├── train-labels-idx1-ubyte (60000个训练标签)\n└── test\n    ├── t10k-images-idx3-ubyte (10000个测试图片)\n    ├── t10k-labels-idx1-ubyte (10000个测试标签)\n",[196],{"type":18,"tag":152,"props":197,"children":198},{"__ignoreMap":7},[199],{"type":24,"value":194},{"type":18,"tag":123,"props":201,"children":203},{"id":202},"_43-数据变换-transforms",[204],{"type":24,"value":205},"4.3 数据变换 Transforms：",{"type":18,"tag":26,"props":207,"children":208},{},[209],{"type":24,"value":210},"数据下载完成后，获得数据集对象。",{"type":18,"tag":147,"props":212,"children":214},{"code":213},"train_dataset = MnistDataset('MNIST_Data/train')\ntest_dataset = MnistDataset('MNIST_Data/test')\n",[215],{"type":18,"tag":152,"props":216,"children":217},{"__ignoreMap":7},[218],{"type":24,"value":213},{"type":18,"tag":26,"props":220,"children":221},{},[222],{"type":24,"value":223},"打印数据集中包含的数据列名，用于dataset的预处理。",{"type":18,"tag":147,"props":225,"children":227},{"code":226},"print(train_dataset.get_col_names())\n['image', 'label']\n",[228],{"type":18,"tag":152,"props":229,"children":230},{"__ignoreMap":7},[231],{"type":24,"value":226},{"type":18,"tag":26,"props":233,"children":234},{},[235],{"type":24,"value":236},"MindSpore的dataset使用数据处理流水线（Data Processing Pipeline），需指定map、batch、shuffle等操作。使用map对图像数据及标签进行变换处理，将输入的图像缩放为1/255，根据均值0.1307和标准差值0.3081进行归一化处理。",{"type":18,"tag":147,"props":238,"children":240},{"code":239},"def datapipe(dataset, batch_size):\n    image_transforms = [\n        vision.Rescale(1.0 / 255.0, 0),\n        vision.Normalize(mean=(0.1307,), std=(0.3081,)),\n        vision.HWC2CHW()\n    ]\n    label_transform = transforms.TypeCast(mindspore.int32)\n\n    dataset = dataset.map(image_transforms, 'image')\n    dataset = dataset.map(label_transform, 'label')\n    dataset = dataset.batch(batch_size)\n    return dataset\n",[241],{"type":18,"tag":152,"props":242,"children":243},{"__ignoreMap":7},[244],{"type":24,"value":239},{"type":18,"tag":26,"props":246,"children":247},{},[248],{"type":24,"value":249},"然后将处理好的数据集打包为大小为64的batch。",{"type":18,"tag":147,"props":251,"children":253},{"code":252},"# Map vision transforms and batch dataset\ntrain_dataset = datapipe(train_dataset, 64)\ntest_dataset = datapipe(test_dataset, 64)\n",[254],{"type":18,"tag":152,"props":255,"children":256},{"__ignoreMap":7},[257],{"type":24,"value":252},{"type":18,"tag":26,"props":259,"children":260},{},[261],{"type":24,"value":262},"可使用create_tuple_iterator 或create_dict_iterator对数据集进行迭代访问，查看数据和标签的shape和datatype。",{"type":18,"tag":147,"props":264,"children":266},{"code":265},"for image, label in test_dataset.create_tuple_iterator():\n    print(f\"Shape of image [N, C, H, W]: {image.shape} {image.dtype}\")\n    print(f\"Shape of label: {label.shape} {label.dtype}\")\n    break\n\nfor data in test_dataset.create_dict_iterator():\n    print(f\"Shape of image [N, C, H, W]: {data['image'].shape} {data['image'].dtype}\")\n    print(f\"Shape of label: {data['label'].shape} {data['label'].dtype}\")\n    break\n",[267],{"type":18,"tag":152,"props":268,"children":269},{"__ignoreMap":7},[270],{"type":24,"value":265},{"type":18,"tag":26,"props":272,"children":273},{},[274],{"type":18,"tag":113,"props":275,"children":277},{"alt":7,"src":276},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/08/31/c5269f0b591a4b04b01156c5ab5d2d1b.png",[],{"type":18,"tag":123,"props":279,"children":281},{"id":280},"_44-网络构建",[282],{"type":24,"value":283},"4.4 网络构建：",{"type":18,"tag":26,"props":285,"children":286},{},[287],{"type":24,"value":288},"mindspore.nn类是构建所有网络的基类，也是网络的基本单元。当用户需要自定义网络时，可以继承nn.Cell类，并重写__init__方法和construct方法。__init__包含所有网络层的定义，construct中包含数据（Tensor）的变换过程。",{"type":18,"tag":147,"props":290,"children":292},{"code":291},"# Define model\nclass Network(nn.Cell):\n    def __init__(self):\n        super().__init__()\n        self.flatten = nn.Flatten()\n        self.dense_relu_sequential = nn.SequentialCell(\n            nn.Dense(28*28, 512),\n            nn.ReLU(),\n            nn.Dense(512, 512),\n            nn.ReLU(),\n            nn.Dense(512, 10)\n        )\n\n    def construct(self, x):\n        x = self.flatten(x)\n        logits = self.dense_relu_sequential(x)\n        return logits\n\nmodel = Network()\nprint(model)\n",[293],{"type":18,"tag":152,"props":294,"children":295},{"__ignoreMap":7},[296],{"type":24,"value":291},{"type":18,"tag":26,"props":298,"children":299},{},[300],{"type":18,"tag":113,"props":301,"children":303},{"alt":7,"src":302},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/08/31/7be08f365f784691b3ff90b857d0aabe.png",[],{"type":18,"tag":123,"props":305,"children":307},{"id":306},"_44-模型训练",[308],{"type":24,"value":309},"4.4 模型训练：",{"type":18,"tag":26,"props":311,"children":312},{},[313],{"type":24,"value":314},"在模型训练中，一个完整的训练过程（step）需要实现以下三步：",{"type":18,"tag":316,"props":317,"children":318},"ul",{},[319,325,330],{"type":18,"tag":320,"props":321,"children":322},"li",{},[323],{"type":24,"value":324},"①. 正向计算：模型预测结果（logits），并与正确标签（label）求预测损失（loss）。",{"type":18,"tag":320,"props":326,"children":327},{},[328],{"type":24,"value":329},"②. 反向传播：利用自动微分机制，自动求模型参数（parameters）对于loss的梯度（gradients）。",{"type":18,"tag":320,"props":331,"children":332},{},[333],{"type":24,"value":334},"③. 参数优化：将梯度更新到参数上。",{"type":18,"tag":26,"props":336,"children":337},{},[338],{"type":24,"value":339},"MindSpore使用函数式自动微分机制，因此针对上述步骤需要实现：",{"type":18,"tag":316,"props":341,"children":342},{},[343,348],{"type":18,"tag":320,"props":344,"children":345},{},[346],{"type":24,"value":347},"①. 定义正向计算函数。",{"type":18,"tag":320,"props":349,"children":350},{},[351],{"type":24,"value":352},"②. 使用value_and_grad通过函数变换获得梯度计算函数。",{"type":18,"tag":26,"props":354,"children":355},{},[356],{"type":24,"value":357},"定义训练函数，使用set_train设置为训练模式，执行正向计算、反向传播和参数优化。",{"type":18,"tag":147,"props":359,"children":361},{"code":360},"# Instantiate loss function and optimizer\nloss_fn = nn.CrossEntropyLoss()\noptimizer = nn.SGD(model.trainable_params(), 1e-2)\n\n# 1. Define forward function\ndef forward_fn(data, label):\n    logits = model(data)\n    loss = loss_fn(logits, label)\n    return loss, logits\n\n# 2. Get gradient function\ngrad_fn = mindspore.value_and_grad(forward_fn, None, optimizer.parameters, has_aux=True)\n\n# 3. Define function of one-step training\ndef train_step(data, label):\n    (loss, _), grads = grad_fn(data, label)\n    optimizer(grads)\n    return loss\n\ndef train(model, dataset):\n    size = dataset.get_dataset_size()\n    model.set_train()\n    for batch, (data, label) in enumerate(dataset.create_tuple_iterator()):\n        loss = train_step(data, label)\n\n        if batch % 100 == 0:\n            loss, current = loss.asnumpy(), batch\n            print(f\"loss: {loss:>7f}  [{current:>3d}/{size:>3d}]\")\n",[362],{"type":18,"tag":152,"props":363,"children":364},{"__ignoreMap":7},[365],{"type":24,"value":360},{"type":18,"tag":26,"props":367,"children":368},{},[369],{"type":24,"value":370},"除训练外，我们定义测试函数，用来评估模型的性能。",{"type":18,"tag":147,"props":372,"children":374},{"code":373},"def test(model, dataset, loss_fn):\n    num_batches = dataset.get_dataset_size()\n    model.set_train(False)\n    total, test_loss, correct = 0, 0, 0\n    for data, label in dataset.create_tuple_iterator():\n        pred = model(data)\n        total += len(data)\n        test_loss += loss_fn(pred, label).asnumpy()\n        correct += (pred.argmax(1) == label).asnumpy().sum()\n    test_loss /= num_batches\n    correct /= total\n    print(f\"Test: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",[375],{"type":18,"tag":152,"props":376,"children":377},{"__ignoreMap":7},[378],{"type":24,"value":373},{"type":18,"tag":26,"props":380,"children":381},{},[382],{"type":24,"value":383},"训练过程需多次迭代数据集，一次完整的迭代称为一轮（epoch）。在每一轮，遍历训练集进行训练，结束后使用测试集进行预测。打印每一轮的loss值和预测准确率（Accuracy），可以看到loss在不断下降，Accuracy在不断提高。",{"type":18,"tag":26,"props":385,"children":386},{},[387],{"type":18,"tag":113,"props":388,"children":390},{"alt":7,"src":389},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/08/31/ec9a81ff93c24445b70e87dc1e040c7e.png",[],{"type":18,"tag":123,"props":392,"children":394},{"id":393},"_45-保存模型",[395],{"type":24,"value":396},"4.5 保存模型：",{"type":18,"tag":26,"props":398,"children":399},{},[400],{"type":24,"value":401},"模型训练完成后，需要将其参数进行保存。",{"type":18,"tag":147,"props":403,"children":405},{"code":404},"# Save checkpoint\nmindspore.save_checkpoint(model, \"model.ckpt\")\nprint(\"Saved Model to model.ckpt\")\nSaved Model to model.ckpt\n",[406],{"type":18,"tag":152,"props":407,"children":408},{"__ignoreMap":7},[409],{"type":24,"value":404},{"type":18,"tag":123,"props":411,"children":413},{"id":412},"_46-加载模型",[414],{"type":24,"value":415},"4.6 加载模型：",{"type":18,"tag":26,"props":417,"children":418},{},[419],{"type":24,"value":420},"加载保存的权重分为两步：",{"type":18,"tag":316,"props":422,"children":423},{},[424,429],{"type":18,"tag":320,"props":425,"children":426},{},[427],{"type":24,"value":428},"重新实例化模型对象，构造模型。",{"type":18,"tag":320,"props":430,"children":431},{},[432],{"type":24,"value":433},"加载模型参数，并将其加载至模型上。",{"type":18,"tag":147,"props":435,"children":437},{"code":436},"# Instantiate a random initialized model\nmodel = Network()\n# Load checkpoint and load parameter to model\nparam_dict = mindspore.load_checkpoint(\"model.ckpt\")\nparam_not_load, _ = mindspore.load_param_into_net(model, param_dict)\nprint(param_not_load)\n",[438],{"type":18,"tag":152,"props":439,"children":440},{"__ignoreMap":7},[441],{"type":24,"value":436},{"type":18,"tag":26,"props":443,"children":444},{},[445],{"type":24,"value":446},"param_not_load是未被加载的参数列表，为空时代表所有参数均加载成功。",{"type":18,"tag":26,"props":448,"children":449},{},[450],{"type":24,"value":451},"加载后的模型可以直接用于预测推理。",{"type":18,"tag":147,"props":453,"children":455},{"code":454},"model.set_train(False)\nfor data, label in test_dataset:\n    pred = model(data)\n    predicted = pred.argmax(1)\n    print(f'Predicted: \"{predicted[:10]}\", Actual: \"{label[:10]}\"')\n    break\n",[456],{"type":18,"tag":152,"props":457,"children":458},{"__ignoreMap":7},[459],{"type":24,"value":454},{"type":18,"tag":316,"props":461,"children":462},{},[463,470,481,486,491,496,501,508],{"type":18,"tag":320,"props":464,"children":465},{},[466],{"type":18,"tag":113,"props":467,"children":469},{"alt":7,"src":468},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/08/31/4679adc6862e412a8771fdb970cd86ab.png",[],{"type":18,"tag":320,"props":471,"children":472},{},[473,479],{"type":18,"tag":123,"props":474,"children":476},{"id":475},"_46-总结",[477],{"type":24,"value":478},"4.6 总结：",{"type":24,"value":480},"上面我们通过MindSpore的API来快速实现一个简单的深度学习模型，从加载数据集、数据变换 Transforms、网络构建、模型训练、模型保存、加载模型，完成了一个简单的深度学习模型的闭环，可以体验到：",{"type":18,"tag":320,"props":482,"children":483},{},[484],{"type":24,"value":485},"①. 能过jupyter云上开发环境，我们可以不用搭建自己的AI应用。",{"type":18,"tag":320,"props":487,"children":488},{},[489],{"type":24,"value":490},"②. MindSpore的API简化了大量深度学习的复杂度。",{"type":18,"tag":320,"props":492,"children":493},{},[494],{"type":24,"value":495},"③. 通过使用华为云可以达到百M速度的下载，十分的高效。",{"type":18,"tag":320,"props":497,"children":498},{},[499],{"type":24,"value":500},"④. 在模型训练中，可以在几分钟内就完成模型的训练，大大的简化了实验中等待的过程。",{"type":18,"tag":320,"props":502,"children":503},{},[504],{"type":18,"tag":113,"props":505,"children":507},{"alt":7,"src":506},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/08/31/208bfd683b78451ba6b8f912afbd036b.png",[],{"type":18,"tag":320,"props":509,"children":510},{},[511,513],{"type":24,"value":512},"接下来，大家可以体验一下其它的入门学习课程，下一步将带大家体验一个实战的垃圾识别场景。",{"type":18,"tag":96,"props":514,"children":515},{},[516],{"type":18,"tag":26,"props":517,"children":518},{},[519],{"type":24,"value":520},"如本文档以Notebook运行时，完成安装后需要重启kernel才能执行后续代码。",{"title":7,"searchDepth":522,"depth":522,"links":523},4,[524,526,527,528,529,530,531,532],{"id":125,"depth":525,"text":128},2,{"id":158,"depth":525,"text":161},{"id":202,"depth":525,"text":205},{"id":280,"depth":525,"text":283},{"id":306,"depth":525,"text":309},{"id":393,"depth":525,"text":396},{"id":412,"depth":525,"text":415},{"id":475,"depth":525,"text":478},"markdown","content:technology-blogs:zh:3293.md","content","technology-blogs/zh/3293.md","technology-blogs/zh/3293","md",1776506128681]