[{"data":1,"prerenderedAt":350},["ShallowReactive",2],{"content-query-csByeM95wU":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":344,"_id":345,"_source":346,"_file":347,"_stem":348,"_extension":349},"/technology-blogs/zh/3255","zh",false,"","女朋友又不开心了？基于昇思MindSpore实现BERT对话情绪识别","作者：JeffDing 来源：昇思社区论坛","2024-08-06","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2024/08/09/374f3f4869de41a291651b8172763326.png","technology-blogs","实践",{"type":15,"children":16,"toc":341},"root",[17,25,44,52,60,65,70,75,80,85,90,98,106,116,121,129,137,145,150,155,160,165,170,175,183,191,199,204,212,217,225,233,241,246,254,262,270,275,283,291,299,304,312,320,328,333],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"女朋友又不开心了基于昇思mindspore实现bert对话情绪识别",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":28},"p",{},[29,31,37,39],{"type":24,"value":30},"**作者：**",{"type":18,"tag":32,"props":33,"children":34},"strong",{},[35],{"type":24,"value":36},"JeffDing",{"type":24,"value":38}," ",{"type":18,"tag":32,"props":40,"children":41},{},[42],{"type":24,"value":43},"来源：昇思社区论坛",{"type":18,"tag":26,"props":45,"children":46},{},[47],{"type":18,"tag":32,"props":48,"children":49},{},[50],{"type":24,"value":51},"01",{"type":18,"tag":26,"props":53,"children":54},{},[55],{"type":18,"tag":32,"props":56,"children":57},{},[58],{"type":24,"value":59},"模型简介",{"type":18,"tag":26,"props":61,"children":62},{},[63],{"type":24,"value":64},"BERT全称是来自变换器的双向编码器表征量（Bidirectional Encoder Representations from Transformers），它是Google于2018年末开发并发布的一种新型语言模型。与BERT模型相似的预训练语言模型例如问答、命名实体识别、自然语言推理、文本分类等在许多自然语言处理任务中发挥着重要作用。模型是基于Transformer中的Encoder并加上双向的结构，因此一定要熟练掌握Transformer的Encoder的结构。",{"type":18,"tag":26,"props":66,"children":67},{},[68],{"type":24,"value":69},"BERT模型的主要创新点都在pre-train方法上，即用了Masked Language Model和Next Sentence Prediction两种方法分别捕捉词语和句子级别的representation。",{"type":18,"tag":26,"props":71,"children":72},{},[73],{"type":24,"value":74},"在用Masked Language Model方法训练BERT的时候，随机把语料库中15%的单词做Mask操作。对于这15%的单词做Mask操作分为三种情况：80%的单词直接用[Mask]替换、10%的单词直接替换成另一个新的单词、10%的单词保持不变。",{"type":18,"tag":26,"props":76,"children":77},{},[78],{"type":24,"value":79},"因为涉及到Question Answering (QA) 和 Natural Language Inference (NLI)之类的任务，增加了Next Sentence Prediction预训练任务，目的是让模型理解两个句子之间的联系。与Masked Language Model任务相比，Next Sentence Prediction更简单些，训练的输入是句子A和B，B有一半的几率是A的下一句，输入这两个句子，BERT模型预测B是不是A的下一句。",{"type":18,"tag":26,"props":81,"children":82},{},[83],{"type":24,"value":84},"BERT预训练之后，会保存它的Embedding table和12层Transformer权重（BERT-BASE）或24层Transformer权重（BERT-LARGE）。使用预训练好的BERT模型可以对下游任务进行Fine-tuning，比如：文本分类、相似度判断、阅读理解等。",{"type":18,"tag":26,"props":86,"children":87},{},[88],{"type":24,"value":89},"对话情绪识别（Emotion Detection，简称EmoTect），专注于识别智能对话场景中用户的情绪，针对智能对话场景中的用户文本，自动判断该文本的情绪类别并给出相应的置信度，情绪类型分为积极、消极、中性。对话情绪识别适用于聊天、客服等多个场景，能够帮助企业更好地把握对话质量、改善产品的用户交互体验，也能分析客服服务质量、降低人工质检成本。",{"type":18,"tag":26,"props":91,"children":92},{},[93],{"type":18,"tag":32,"props":94,"children":95},{},[96],{"type":24,"value":97},"02",{"type":18,"tag":26,"props":99,"children":100},{},[101],{"type":18,"tag":32,"props":102,"children":103},{},[104],{"type":24,"value":105},"安装mindnlp",{"type":18,"tag":107,"props":108,"children":110},"pre",{"code":109},"pip install mindnlp\n",[111],{"type":18,"tag":112,"props":113,"children":114},"code",{"__ignoreMap":7},[115],{"type":24,"value":109},{"type":18,"tag":26,"props":117,"children":118},{},[119],{"type":24,"value":120},"下面以一个文本情感分类任务为例子来说明BERT模型的整个应用过程。",{"type":18,"tag":107,"props":122,"children":124},{"code":123},"import os\n\nimport mindspore\nfrom mindspore.dataset import text, GeneratorDataset, transforms\nfrom mindspore import nn, context\n\nfrom mindnlp._legacy.engine import Trainer, Evaluator\nfrom mindnlp._legacy.engine.callbacks import CheckpointCallback, BestModelCallback\nfrom mindnlp._legacy.metrics import Accuracy\n\n# prepare dataset\nclass SentimentDataset:\n    \"\"\"Sentiment Dataset\"\"\"\n\n    def __init__(self, path):\n        self.path = path\n        self._labels, self._text_a = [], []\n        self._load()\n\n    def _load(self):\n        with open(self.path, \"r\", encoding=\"utf-8\") as f:\n            dataset = f.read()\n        lines = dataset.split(\"\\n\")\n        for line in lines[1:-1]:\n            label, text_a = line.split(\"\\t\")\n            self._labels.append(int(label))\n            self._text_a.append(text_a)\n\n    def __getitem__(self, index):\n        return self._labels[index], self._text_a[index]\n\n    def __len__(self):\n        return len(self._labels)\n",[125],{"type":18,"tag":112,"props":126,"children":127},{"__ignoreMap":7},[128],{"type":24,"value":123},{"type":18,"tag":26,"props":130,"children":131},{},[132],{"type":18,"tag":32,"props":133,"children":134},{},[135],{"type":24,"value":136},"03",{"type":18,"tag":26,"props":138,"children":139},{},[140],{"type":18,"tag":32,"props":141,"children":142},{},[143],{"type":24,"value":144},"数据集",{"type":18,"tag":26,"props":146,"children":147},{},[148],{"type":24,"value":149},"这里提供一份已标注的、经过分词预处理的机器人聊天数据集。数据由两列组成，以制表符（’\\t’）分隔，第一列是情绪分类的类别（0表示消极；1表示中性；2表示积极），第二列是以空格分词的中文文本，如下示例，文件为utf8编码。",{"type":18,"tag":26,"props":151,"children":152},{},[153],{"type":24,"value":154},"label–text_a",{"type":18,"tag":26,"props":156,"children":157},{},[158],{"type":24,"value":159},"0–谁骂人了？我从来不骂人，我骂的都不是人，你是人吗 ？",{"type":18,"tag":26,"props":161,"children":162},{},[163],{"type":24,"value":164},"1–我有事等会儿就回来和你聊",{"type":18,"tag":26,"props":166,"children":167},{},[168],{"type":24,"value":169},"2–我见到你很高兴谢谢你帮我",{"type":18,"tag":26,"props":171,"children":172},{},[173],{"type":24,"value":174},"这部分主要包括数据集读取，数据格式转换，数据Tokenize处理和pad操作。",{"type":18,"tag":107,"props":176,"children":178},{"code":177},"wget https://baidu-nlp.bj.bcebos.com/emotion_detection-dataset-1.0.0.tar.gz -O emotion_detection.tar.gz\ntar xvf emotion_detection.tar.gz\n",[179],{"type":18,"tag":112,"props":180,"children":181},{"__ignoreMap":7},[182],{"type":24,"value":177},{"type":18,"tag":26,"props":184,"children":185},{},[186],{"type":18,"tag":32,"props":187,"children":188},{},[189],{"type":24,"value":190},"04",{"type":18,"tag":26,"props":192,"children":193},{},[194],{"type":18,"tag":32,"props":195,"children":196},{},[197],{"type":24,"value":198},"数据加载和数据预处理",{"type":18,"tag":26,"props":200,"children":201},{},[202],{"type":24,"value":203},"新建process_dataset函数用于数据加载和数据预处理，具体内容可见下面代码注释。",{"type":18,"tag":107,"props":205,"children":207},{"code":206},"import numpy as np\n\ndef process_dataset(source, tokenizer, max_seq_len=64, batch_size=32, shuffle=True):\n    is_ascend = mindspore.get_context('device_target') == 'Ascend'\n\n    column_names = [\"label\", \"text_a\"]\n\n    dataset = GeneratorDataset(source, column_names=column_names, shuffle=shuffle)\n    # transforms\n    type_cast_op = transforms.TypeCast(mindspore.int32)\n    def tokenize_and_pad(text):\n        if is_ascend:\n            tokenized = tokenizer(text, padding='max_length', truncation=True, max_length=max_seq_len)\n        else:\n            tokenized = tokenizer(text)\n        return tokenized['input_ids'], tokenized['attention_mask']\n    # map dataset\n    dataset = dataset.map(operations=tokenize_and_pad, input_columns=\"text_a\", output_columns=['input_ids', 'attention_mask'])\n    dataset = dataset.map(operations=[type_cast_op], input_columns=\"label\", output_columns='labels')\n    # batch dataset\n    if is_ascend:\n        dataset = dataset.batch(batch_size)\n    else:\n        dataset = dataset.padded_batch(batch_size, pad_info={'input_ids': (None, tokenizer.pad_token_id),\n                                                         'attention_mask': (None, 0)})\n\n    return dataset\n",[208],{"type":18,"tag":112,"props":209,"children":210},{"__ignoreMap":7},[211],{"type":24,"value":206},{"type":18,"tag":26,"props":213,"children":214},{},[215],{"type":24,"value":216},"数据预处理部分采用静态Shape处理：",{"type":18,"tag":107,"props":218,"children":220},{"code":219},"from mindnlp.transformers import BertTokenizer\ntokenizer = BertTokenizer.from_pretrained('bert-base-chinese')\n\ntokenizer.pad_token_id\n\ndataset_train = process_dataset(SentimentDataset(\"data/train.tsv\"), tokenizer)\ndataset_val = process_dataset(SentimentDataset(\"data/dev.tsv\"), tokenizer)\ndataset_test = process_dataset(SentimentDataset(\"data/test.tsv\"), tokenizer, shuffle=False)\n\nprint(next(dataset_train.create_tuple_iterator()))\n",[221],{"type":18,"tag":112,"props":222,"children":223},{"__ignoreMap":7},[224],{"type":24,"value":219},{"type":18,"tag":26,"props":226,"children":227},{},[228],{"type":18,"tag":32,"props":229,"children":230},{},[231],{"type":24,"value":232},"05",{"type":18,"tag":26,"props":234,"children":235},{},[236],{"type":18,"tag":32,"props":237,"children":238},{},[239],{"type":24,"value":240},"模型构建",{"type":18,"tag":26,"props":242,"children":243},{},[244],{"type":24,"value":245},"通过 BertForSequenceClassification 构建用于情感分类的BERT模型，加载预训练权重，设置情感三分类的超参数自动构建模型。后面对模型采用自动混合精度操作，提高训练的速度，然后实例化优化器，紧接着实例化评价指标，设置模型训练的权重保存策略，最后就是构建训练器，模型开始训练。",{"type":18,"tag":107,"props":247,"children":249},{"code":248},"from mindnlp.transformers import BertForSequenceClassification, BertModel\nfrom mindnlp._legacy.amp import auto_mixed_precision\n\n# set bert config and define parameters for training\nmodel = BertForSequenceClassification.from_pretrained('bert-base-chinese', num_labels=3)\nmodel = auto_mixed_precision(model, 'O1')\n\noptimizer = nn.Adam(model.trainable_params(), learning_rate=2e-5)\n\nmetric = Accuracy()\n# define callbacks to save checkpoints\nckpoint_cb = CheckpointCallback(save_path='checkpoint', ckpt_name='bert_emotect', epochs=1, keep_checkpoint_max=2)\nbest_model_cb = BestModelCallback(save_path='checkpoint', ckpt_name='bert_emotect_best', auto_load=True)\n\ntrainer = Trainer(network=model, train_dataset=dataset_train,\n                  eval_dataset=dataset_val, metrics=metric,\n                  epochs=5, optimizer=optimizer, callbacks=[ckpoint_cb, best_model_cb])\n# start training\ntrainer.run(tgt_columns=\"labels\")\n",[250],{"type":18,"tag":112,"props":251,"children":252},{"__ignoreMap":7},[253],{"type":24,"value":248},{"type":18,"tag":26,"props":255,"children":256},{},[257],{"type":18,"tag":32,"props":258,"children":259},{},[260],{"type":24,"value":261},"06",{"type":18,"tag":26,"props":263,"children":264},{},[265],{"type":18,"tag":32,"props":266,"children":267},{},[268],{"type":24,"value":269},"模型验证",{"type":18,"tag":26,"props":271,"children":272},{},[273],{"type":24,"value":274},"将验证数据集加再进训练好的模型，对数据集进行验证，查看模型在验证数据上面的效果，此处的评价指标为准确率。",{"type":18,"tag":107,"props":276,"children":278},{"code":277},"evaluator = Evaluator(network=model, eval_dataset=dataset_test, metrics=metric)\nevaluator.run(tgt_columns=\"labels\")\n",[279],{"type":18,"tag":112,"props":280,"children":281},{"__ignoreMap":7},[282],{"type":24,"value":277},{"type":18,"tag":26,"props":284,"children":285},{},[286],{"type":18,"tag":32,"props":287,"children":288},{},[289],{"type":24,"value":290},"07",{"type":18,"tag":26,"props":292,"children":293},{},[294],{"type":18,"tag":32,"props":295,"children":296},{},[297],{"type":24,"value":298},"模型推理",{"type":18,"tag":26,"props":300,"children":301},{},[302],{"type":24,"value":303},"遍历推理数据集，将结果与标签进行统一展示。",{"type":18,"tag":107,"props":305,"children":307},{"code":306},"dataset_infer = SentimentDataset(\"data/infer.tsv\")\n\ndef predict(text, label=None):\n    label_map = {0: \"消极\", 1: \"中性\", 2: \"积极\"}\n\n    text_tokenized = Tensor([tokenizer(text).input_ids])\n    logits = model(text_tokenized)\n    predict_label = logits[0].asnumpy().argmax()\n    info = f\"inputs: '{text}', predict: '{label_map[predict_label]}'\"\n    if label is not None:\n        info += f\" , label: '{label_map[label]}'\"\n    print(info)\n\nfrom mindspore import Tensor\n\nfor label, text in dataset_infer:\n    predict(text, label)\n",[308],{"type":18,"tag":112,"props":309,"children":310},{"__ignoreMap":7},[311],{"type":24,"value":306},{"type":18,"tag":26,"props":313,"children":314},{},[315],{"type":18,"tag":32,"props":316,"children":317},{},[318],{"type":24,"value":319},"08",{"type":18,"tag":26,"props":321,"children":322},{},[323],{"type":18,"tag":32,"props":324,"children":325},{},[326],{"type":24,"value":327},"自定义推理数据集",{"type":18,"tag":26,"props":329,"children":330},{},[331],{"type":24,"value":332},"自己输入推理数据，展示模型的泛化能力。",{"type":18,"tag":107,"props":334,"children":336},{"code":335},"predict(\"家人们咱就是说一整个无语住了 绝绝子叠buff\")\n",[337],{"type":18,"tag":112,"props":338,"children":339},{"__ignoreMap":7},[340],{"type":24,"value":335},{"title":7,"searchDepth":342,"depth":342,"links":343},4,[],"markdown","content:technology-blogs:zh:3255.md","content","technology-blogs/zh/3255.md","technology-blogs/zh/3255","md",1776506128012]