[{"data":1,"prerenderedAt":366},["ShallowReactive",2],{"content-query-44d2jbKthc":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"cover":11,"type":12,"category":13,"body":14,"_type":360,"_id":361,"_source":362,"_file":363,"_stem":364,"_extension":365},"/technology-blogs/zh/780","zh",false,"","技术干货｜昇思MindSpore NLP模型迁移之Bert模型—文本匹配任务（二）：训练和评估","如何使用MindSpore的Bert模型来做下游任务","2021-11-09","https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/11/09/053f25afd71b4b9eae4ae14004716e2e.png","technology-blogs","开发者分享",{"type":15,"children":16,"toc":351},"root",[17,25,34,43,53,58,63,68,73,78,83,88,93,98,103,108,115,120,125,135,140,148,153,160,169,174,182,191,203,211,220,228,233,238,243,248,253,260,265,273,286,291,298,308,313,320,323,328,341,346],{"type":18,"tag":19,"props":20,"children":22},"element","h1",{"id":21},"技术干货昇思mindspore-nlp模型迁移之bert模型文本匹配任务二训练和评估",[23],{"type":24,"value":8},"text",{"type":18,"tag":26,"props":27,"children":28},"p",{},[29],{"type":18,"tag":30,"props":31,"children":33},"img",{"alt":7,"src":32},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/11/09/cd05c637210245d7bd48acb8b70180bb.gif",[],{"type":18,"tag":26,"props":35,"children":36},{},[37],{"type":18,"tag":38,"props":39,"children":40},"strong",{},[41],{"type":24,"value":42},"作者：Lil J ｜来源：知乎",{"type":18,"tag":44,"props":45,"children":47},"h3",{"id":46},"前言",[48],{"type":18,"tag":38,"props":49,"children":50},{},[51],{"type":24,"value":52},"前言:",{"type":18,"tag":26,"props":54,"children":55},{},[56],{"type":24,"value":57},"在上一节中，我们已经将Bert模型迁移到MindSpore上。在这节中，我将会介绍如何使用MindSpore的Bert模型来做下游任务：lcqmc的文本匹配任务。",{"type":18,"tag":26,"props":59,"children":60},{},[61],{"type":24,"value":62},"主机环境：",{"type":18,"tag":26,"props":64,"children":65},{},[66],{"type":24,"value":67},"系统：ubuntu18",{"type":18,"tag":26,"props":69,"children":70},{},[71],{"type":24,"value":72},"GPU：3090",{"type":18,"tag":26,"props":74,"children":75},{},[76],{"type":24,"value":77},"MindSpore版本：1.3",{"type":18,"tag":26,"props":79,"children":80},{},[81],{"type":24,"value":82},"数据集：lcqmc",{"type":18,"tag":26,"props":84,"children":85},{},[86],{"type":24,"value":87},"lcqmc文本匹配任务的定义：",{"type":18,"tag":26,"props":89,"children":90},{},[91],{"type":24,"value":92},"哈工大文本匹配数据集，LCQMC 是哈尔滨工业大学在自然语言处理国际顶会 COLING2018 构建的问题语义匹配数据集，其目标是判断两个问题的语义是否相同。",{"type":18,"tag":26,"props":94,"children":95},{},[96],{"type":24,"value":97},"数据集中的字段分别如下：",{"type":18,"tag":26,"props":99,"children":100},{},[101],{"type":24,"value":102},"text_a， text_b, label。",{"type":18,"tag":26,"props":104,"children":105},{},[106],{"type":24,"value":107},"其中text_a和text_b为两个问题的文本。若两个问题的语义相同则label为1，否则为0。",{"type":18,"tag":26,"props":109,"children":110},{},[111],{"type":18,"tag":30,"props":112,"children":114},{"alt":7,"src":113},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/11/09/f5f6dcea4d49471ea5aa3658586871c5.png",[],{"type":18,"tag":26,"props":116,"children":117},{},[118],{"type":24,"value":119},"由于官网已经提供了微调好的权重信息，所以我们尝试直接转换权重进行预测。我们先要知道模型权重名称以及形状等，需要PyTorch与MindSpore模型一一对应。",{"type":18,"tag":26,"props":121,"children":122},{},[123],{"type":24,"value":124},"首先，我们将huggingface的bert-chinese-base的torch bin文件下载下来。接下来使用下面的函数将Torch权重参数文件转化为MindSpore权重参数文件",{"type":18,"tag":126,"props":127,"children":129},"pre",{"code":128},"def torch_to_ms(model, torch_model,save_path):\n    \"\"\"\n    Updates mobilenetv2 model mindspore param's data from torch param's data.\n    Args:\n        model: mindspore model\n        torch_model: torch model\n    \"\"\"\n    print(\"start load\")\n    # load torch parameter and mindspore parameter\n    torch_param_dict = torch_model\n    ms_param_dict = model.parameters_dict()\n    count = 0\n    for ms_key in ms_param_dict.keys():\n        ms_key_tmp = ms_key.split('.')\n        if ms_key_tmp[0] == 'bert_embedding_lookup':\n            count+=1\n            update_torch_to_ms(torch_param_dict, ms_param_dict, 'embeddings.word_embeddings.weight', ms_key)\n        elif ms_key_tmp[0] == 'bert_embedding_postprocessor':\n            if ms_key_tmp[1] == \"token_type_embedding\":\n                count+=1\n                update_torch_to_ms(torch_param_dict, ms_param_dict, 'embeddings.token_type_embeddings.weight', ms_key)\n            elif ms_key_tmp[1] == \"full_position_embedding\":\n                count+=1\n                update_torch_to_ms(torch_param_dict, ms_param_dict, 'embeddings.position_embeddings.weight',\n                                   ms_key)\n            elif ms_key_tmp[1] ==\"layernorm\":\n                if ms_key_tmp[2]==\"gamma\":\n                    count+=1\n                    update_torch_to_ms(torch_param_dict, ms_param_dict, 'embeddings.LayerNorm.weight',\n                                       ms_key)\n                else:\n                    count+=1\n                    update_torch_to_ms(torch_param_dict, ms_param_dict, 'embeddings.LayerNorm.bias',\n                                       ms_key)\n        elif ms_key_tmp[0] == \"bert_encoder\":\n            if ms_key_tmp[3] == 'attention':\n                    par = ms_key_tmp[4].split('_')[0]\n                    count+=1\n                    update_torch_to_ms(torch_param_dict, ms_param_dict, 'encoder.layer.'+ms_key_tmp[2]+'.'+ms_key_tmp[3]+'.'\n                                       +'self.'+par+'.'+ms_key_tmp[5],\n                                       ms_key)\n            elif ms_key_tmp[3] == 'attention_output':\n                if ms_key_tmp[4] == 'dense':\n                    print(7)\n                    count+=1\n                    update_torch_to_ms(torch_param_dict, ms_param_dict,\n                                   'encoder.layer.' + ms_key_tmp[2] + '.attention.output.'+ms_key_tmp[4]+'.'+ms_key_tmp[5],\n                                   ms_key)\n\n                elif ms_key_tmp[4]=='layernorm':\n                    if ms_key_tmp[5]=='gamma':\n                        print(8)\n                        count+=1\n                        update_torch_to_ms(torch_param_dict, ms_param_dict,\n                                           'encoder.layer.' + ms_key_tmp[2] + '.attention.output.LayerNorm.weight',\n                                           ms_key)\n                    else:\n                        count+=1\n                        update_torch_to_ms(torch_param_dict, ms_param_dict,\n                                           'encoder.layer.' + ms_key_tmp[2] + '.attention.output.LayerNorm.bias',\n                                           ms_key)\n            elif ms_key_tmp[3] == 'intermediate':\n                count+=1\n                update_torch_to_ms(torch_param_dict, ms_param_dict,\n                                   'encoder.layer.' + ms_key_tmp[2] + '.intermediate.dense.'+ms_key_tmp[4],\n                                   ms_key)\n            elif ms_key_tmp[3] == 'output':\n                if ms_key_tmp[4] == 'dense':\n                    count+=1\n                    update_torch_to_ms(torch_param_dict, ms_param_dict,\n                                   'encoder.layer.' + ms_key_tmp[2] + '.output.dense.'+ms_key_tmp[5],\n                                   ms_key)\n                else:\n                    if ms_key_tmp[5] == 'gamma':\n                        count+=1\n                        update_torch_to_ms(torch_param_dict, ms_param_dict,\n                                       'encoder.layer.' + ms_key_tmp[2] + '.output.LayerNorm.weight',\n                                       ms_key)\n\n                    else:\n                        count+=1\n                        update_torch_to_ms(torch_param_dict, ms_param_dict,\n                                       'encoder.layer.' + ms_key_tmp[2] + '.output.LayerNorm.bias',\n                                       ms_key)\n\n        if ms_key_tmp[0] == 'dense':\n            if ms_key_tmp[1] == 'weight':\n                count+=1\n                update_torch_to_ms(torch_param_dict, ms_param_dict,\n                                   'pooler.dense.weight',\n                                   ms_key)\n            else:\n                count+=1\n                update_torch_to_ms(torch_param_dict, ms_param_dict,\n                                   'pooler.dense.bias',\n                                   ms_key)\n\n    save_checkpoint(model, save_path)\n    print(\"finish load\")\n\n\n\ndef update_bn(torch_param_dict, ms_param_dict, ms_key, ms_key_tmp):\n    \"\"\"Updates mindspore batchnorm param's data from torch batchnorm param's data.\"\"\"\n\n    str_join = '.'\n    if ms_key_tmp[-1] == \"moving_mean\":\n        ms_key_tmp[-1] = \"running_mean\"\n        torch_key = str_join.join(ms_key_tmp)\n        update_torch_to_ms(torch_param_dict, ms_param_dict, torch_key, ms_key)\n    elif ms_key_tmp[-1] == \"moving_variance\":\n        ms_key_tmp[-1] = \"running_var\"\n        torch_key = str_join.join(ms_key_tmp)\n        update_torch_to_ms(torch_param_dict, ms_param_dict, torch_key, ms_key)\n    elif ms_key_tmp[-1] == \"gamma\":\n        ms_key_tmp[-1] = \"weight\"\n        torch_key = str_join.join(ms_key_tmp)\n        update_torch_to_ms(torch_param_dict, ms_param_dict, 'transformer.' + torch_key, ms_key)\n    elif ms_key_tmp[-1] == \"beta\":\n        ms_key_tmp[-1] = \"bias\"\n        torch_key = str_join.join(ms_key_tmp)\n        update_torch_to_ms(torch_param_dict, ms_param_dict, 'transformer.' + torch_key, ms_key)\n\n\ndef update_torch_to_ms(torch_param_dict, ms_param_dict, torch_key, ms_key):\n    \"\"\"Updates mindspore param's data from torch param's data.\"\"\"\n\n    value = torch_param_dict[torch_key].cpu().numpy()\n    value = Parameter(Tensor(value), name=ms_key)\n    _update_param(ms_param_dict[ms_key], value)\n\n\n\ndef _update_param(param, new_param):\n    \"\"\"Updates param's data from new_param's data.\"\"\"\n\n    if isinstance(param.data, Tensor) and isinstance(new_param.data, Tensor):\n        if param.data.dtype != new_param.data.dtype:\n            print(\"Failed to combine the net and the parameters for param %s.\", param.name)\n            msg = (\"Net parameters {} type({}) different from parameter_dict's({})\"\n                   .format(param.name, param.data.dtype, new_param.data.dtype))\n            raise RuntimeError(msg)\n\n        if param.data.shape != new_param.data.shape:\n            if not _special_process_par(param, new_param):\n                print(\"Failed to combine the net and the parameters for param %s.\", param.name)\n                msg = (\"Net parameters {} shape({}) different from parameter_dict's({})\"\n                       .format(param.name, param.data.shape, new_param.data.shape))\n                raise RuntimeError(msg)\n            return\n\n        param.set_data(new_param.data)\n        return\n\n    if isinstance(param.data, Tensor) and not isinstance(new_param.data, Tensor):\n              if param.data.shape != (1,) and param.data.shape != ():\n            print(\"Failed to combine the net and the parameters for param %s.\", param.name)\n            msg = (\"Net parameters {} shape({}) is not (1,), inconsistent with parameter_dict's(scalar).\"\n                   .format(param.name, param.data.shape))\n            raise RuntimeError(msg)\n        param.set_data(initializer(new_param.data, param.data.shape, param.data.dtype))\n\n    elif isinstance(new_param.data, Tensor) and not isinstance(param.data, Tensor):\n        print(\"Failed to combine the net and the parameters for param %s.\", param.name)\n        msg = (\"Net parameters {} type({}) different from parameter_dict's({})\"\n               .format(param.name, type(param.data), type(new_param.data)))\n        raise RuntimeError(msg)\n\n    else:\n        param.set_data(type(param.data)(new_param.data))\n\n\ndef _special_process_par(par, new_par):\n    \"\"\"\n    Processes the special condition.\n\n    Like (12,2048,1,1)->(12,2048), this case is caused by GE 4 dimensions tensor.\n    \"\"\"\n    par_shape_len = len(par.data.shape)\n    new_par_shape_len = len(new_par.data.shape)\n    delta_len = new_par_shape_len - par_shape_len\n    delta_i = 0\n    for delta_i in range(delta_len):\n        if new_par.data.shape[par_shape_len + delta_i] != 1:\n            break\n    if delta_i == delta_len - 1:\n        new_val = new_par.data.asnumpy()\n        new_val = new_val.reshape(par.data.shape)\n        par.set_data(Tensor(new_val, par.data.dtype))\n        return True\n    return False\n",[130],{"type":18,"tag":131,"props":132,"children":133},"code",{"__ignoreMap":7},[134],{"type":24,"value":128},{"type":18,"tag":26,"props":136,"children":137},{},[138],{"type":24,"value":139},"实际应用案例如下：",{"type":18,"tag":126,"props":141,"children":143},{"code":142},"import BertConfig\nimport BertModel as ms_bm\nimport BertModel as tc_bm\nbert_config_file = \"./model/test.yaml\"\nbert_config = BertConfig.from_yaml_file(bert_config_file)\nmodel = ms_bm(bert_config, False)\n\ntorch_model = tc_bm.from_pretrained(\"/content/model/bert_cn\")\ntorch_to_ms(model, torch_model.state_dict(),\"./model/bert2.ckpt\")\n",[144],{"type":18,"tag":131,"props":145,"children":146},{"__ignoreMap":7},[147],{"type":24,"value":142},{"type":18,"tag":26,"props":149,"children":150},{},[151],{"type":24,"value":152},"这里名称一定要一一对应。如果后期改动了模型，也需要在检查一下这个转换函数是否能对应。",{"type":18,"tag":26,"props":154,"children":155},{},[156],{"type":18,"tag":30,"props":157,"children":159},{"alt":7,"src":158},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/11/09/578bb0c0ad9042b3a8314b7a0ea6575b.png",[],{"type":18,"tag":44,"props":161,"children":163},{"id":162},"_1封装bert为bert_embeding",[164],{"type":18,"tag":38,"props":165,"children":166},{},[167],{"type":24,"value":168},"1、封装Bert为bert_embeding",{"type":18,"tag":26,"props":170,"children":171},{},[172],{"type":24,"value":173},"首先我们先将之前构建好的Bert再进行一步封装为bert_embeding",{"type":18,"tag":126,"props":175,"children":177},{"code":176},"# Copyright 2021 Huawei Technologies Co., Ltd\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n\"\"\"Bert Embedding.\"\"\"\nimport logging\nfrom typing import Tuple\nimport mindspore.nn as nn\nfrom mindspore import Tensor\nfrom mindspore.train.serialization import load_checkpoint, load_param_into_net\nimport BertModel, BertConfig\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')\nlogger = logging.getLogger(__name__)\nclass BertEmbedding(nn.Cell):\n    \"\"\"\n    This is a class that loads pre-trained weight files into the model.\n    \"\"\"\n    def __init__(self, bert_config: BertConfig, is_training: bool = False):\n        super(BertEmbedding, self).__init__()\n        self.bert = BertModel(bert_config, is_training)\n\n    def init_bertmodel(self, bert):\n        \"\"\"\n        Manual initialization BertModel\n        \"\"\"\n        self.bert = bert\n\n    def from_pretrain(self, ckpt_file):\n        \"\"\"\n        Load the model parameters from checkpoint\n        \"\"\"\n        param_dict = load_checkpoint(ckpt_file)\n        load_param_into_net(self.bert, param_dict)\n\n    def construct(self, input_ids: Tensor, token_type_ids: Tensor, input_mask: Tensor) -> Tuple[Tensor, Tensor]:\n        \"\"\"\n        Returns the result of the model after loading the pre-training weights\n\n        Args:\n            input_ids (:class:`mindspore.tensor`):A vector containing the transformation of characters\n                into corresponding ids.\n            token_type_ids (:class:`mindspore.tensor`):A vector containing segemnt ids.\n            input_mask (:class:`mindspore.tensor`):the mask for input_ids.\n        Returns:\n            sequence_output:the sequence output .\n            pooled_output:the pooled output of first token:cls..\n        \"\"\"\n        sequence_output, pooled_output, _ = self.bert(input_ids, token_type_ids, input_mask)\n        return sequence_output, pooled_output\n",[178],{"type":18,"tag":131,"props":179,"children":180},{"__ignoreMap":7},[181],{"type":24,"value":176},{"type":18,"tag":44,"props":183,"children":185},{"id":184},"_2下游任务bertforsequenceclassification",[186],{"type":18,"tag":38,"props":187,"children":188},{},[189],{"type":24,"value":190},"2、下游任务：BertforSequenceClassification",{"type":18,"tag":26,"props":192,"children":193},{},[194,196,201],{"type":24,"value":195},"将Bert作为预训练模型，接着在Bert的基础上，取Bert的cls token的embeding作为输入，输入到全连接网络中，这就是",{"type":18,"tag":38,"props":197,"children":198},{},[199],{"type":24,"value":200},"BertforSequenceClassification",{"type":24,"value":202},"。",{"type":18,"tag":126,"props":204,"children":206},{"code":205},"# Copyright 2021 Huawei Technologies Co., Ltd\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n\"\"\"Bert for Sequence Classification script.\"\"\"\n\nimport numpy as np\nimport mindspore.nn as nn\nimport mindspore.ops as ops\nimport mindspore.common.dtype as mstype\nfrom mindspore.common.initializer import TruncatedNormal\nfrom mindspore.nn.learning_rate_schedule import LearningRateSchedule, PolynomialDecayLR, WarmUpLR\nfrom mindspore.context import ParallelMode\nfrom mindspore.common.tensor import Tensor\nfrom mindspore.common.parameter import Parameter\nfrom mindspore.nn.wrap.grad_reducer import DistributedGradReducer\nfrom mindspore.ops import operations as P\nfrom mindspore.ops import functional as F\nfrom mindspore.ops import composite as C\nfrom mindspore.ops import Squeeze\nfrom mindspore.communication.management import get_group_size\nfrom mindspore import context, load_checkpoint, load_param_into_net\nfrom mindspore.common.seed import _get_graph_seed\nfrom bert_embedding import BertEmbedding\n\n\nclass BertforSequenceClassification(nn.Cell):\n    \"\"\"\n    Train interface for classification finetuning task.\n\n    Args:\n        config (Class): Configuration for BertModel.\n        is_training (bool): True for training mode. False for eval mode.\n        num_labels (int): Number of label types.\n        dropout_prob (float): The dropout probability for BertforSequenceClassification.\n        multi_sample_dropout (int): Dropout times per step\n        label_smooth (float): Label Smoothing Regularization\n    \"\"\"\n    def __init__(self, config, is_training, num_labels, dropout_prob=0.0, multi_sample_dropout=1, label_smooth=1):\n        super(BertforSequenceClassification, self).__init__()\n        if not is_training:\n            config.hidden_dropout_prob = 0.0\n            config.hidden_probs_dropout_prob = 0.0\n        self.bert = BertEmbedding(config, is_training)\n        self.cast = P.Cast()\n        self.weight_init = TruncatedNormal(config.initializer_range)\n        self.softmax = nn.Softmax(axis=-1)\n        self.dtype = config.dtype\n        self.num_labels = num_labels\n        self.dense_1 = nn.Dense(config.hidden_size, self.num_labels, weight_init=self.weight_init,\n                                has_bias=True).to_float(mstype.float32)\n        self.dropout_list=[]\n        for count in range(0, multi_sample_dropout):\n            seed0, seed1 = _get_graph_seed(1, \"dropout\")\n            self.dropout_list.append(ops.Dropout(1-dropout_prob, seed0, seed1))\n        self.loss = nn.SoftmaxCrossEntropyWithLogits(sparse=False, reduction=\"mean\")\n        self.squeeze = Squeeze(1)\n        self.num_labels = num_labels\n        self.is_training = is_training\n        self.one_hot = nn.OneHot(depth=num_labels, axis=-1)\n        self.label_smooth = label_smooth\n\n    def from_pretrain(self, ckpt_file):\n        \"\"\"\n        Load the model parameters from checkpoint\n        \"\"\"\n        param_dict = load_checkpoint(ckpt_file)\n        load_param_into_net(self, param_dict)\n\n    def init_embedding(self, embedding):\n        \"\"\"\n        Manual initialization Embedding\n        \"\"\"\n        self.bert = embedding\n\n    def construct(self, input_ids, input_mask, token_type_id, label_ids=0):\n        \"\"\"\n        Classification task\n        \"\"\"\n        _, pooled_output = self.bert(input_ids, token_type_id, input_mask)\n        loss = None\n        if self.is_training:\n            onehot_label = self.one_hot(self.squeeze(label_ids))\n            smooth_label = self.label_smooth * onehot_label + (1-self.label_smooth)/(self.num_labels-1) * (1-onehot_label)\n            for dropout in self.dropout_list:\n                cls, _ = dropout(pooled_output)\n                logits = self.dense_1(cls)\n                temp_loss = self.loss(logits, smooth_label)\n                if loss == None:\n                    loss = temp_loss\n                else:\n                    loss += temp_loss\n            loss = loss/len(self.dropout_list)\n        else:\n            loss = self.dense_1(pooled_output)\n        return loss\n\n\nclass BertLearningRate(LearningRateSchedule):\n    \"\"\"\n    Warmup-decay learning rate for Bert network.\n    \"\"\"\n\n    def __init__(self, learning_rate, end_learning_rate, warmup_steps, decay_steps, power):\n        super(BertLearningRate, self).__init__()\n        self.warmup_flag = False\n        if warmup_steps > 0:\n            self.warmup_flag = True\n            self.warmup_lr = WarmUpLR(learning_rate, warmup_steps)\n        self.decay_lr = PolynomialDecayLR(learning_rate, end_learning_rate, decay_steps, power)\n        self.warmup_steps = Tensor(np.array([warmup_steps]).astype(np.float32))\n\n        self.greater = P.Greater()\n        self.one = Tensor(np.array([1.0]).astype(np.float32))\n        self.cast = P.Cast()\n\n    def construct(self, global_step):\n        decay_lr = self.decay_lr(global_step)\n        if self.warmup_flag:\n            is_warmup = self.cast(self.greater(self.warmup_steps, global_step), mstype.float32)\n            warmup_lr = self.warmup_lr(global_step)\n            lr = (self.one - is_warmup) * decay_lr + is_warmup * warmup_lr\n        else:\n            lr = decay_lr\n        return lr\n\n\nclass BertFinetuneCell(nn.Cell):\n    \"\"\"\n    Especially defined for finetuning where only four inputs tensor are needed.\n\n    Append an optimizer to the training network after that the construct\n    function can be called to create the backward graph.\n\n    Different from the builtin loss_scale wrapper cell, we apply grad_clip before the optimization.\n\n    Args:\n        network (Cell): The training network. Note that loss function should have been added.\n        optimizer (Optimizer): Optimizer for updating the weights.\n        scale_update_cell (Cell): Cell to do the loss scale. Default: None.\n    \"\"\"\n\n    def __init__(self, network, optimizer, scale_update_cell=None):\n\n        super(BertFinetuneCell, self).__init__(auto_prefix=False)\n        self.network = network\n        self.network.set_grad()\n        self.weights = optimizer.parameters\n        self.optimizer = optimizer\n        self.grad = C.GradOperation(get_by_list=True,\n                                    sens_param=True)\n        self.reducer_flag = False\n        self.allreduce = P.AllReduce()\n        self.parallel_mode = context.get_auto_parallel_context(\"parallel_mode\")\n        if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]:\n            self.reducer_flag = True\n        self.grad_reducer = None\n        if self.reducer_flag:\n            mean = context.get_auto_parallel_context(\"gradients_mean\")\n            degree = get_group_size()\n            self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree)\n        self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE)\n        self.cast = P.Cast()\n        self.gpu_target = False\n        if context.get_context(\"device_target\") == \"GPU\":\n            self.gpu_target = True\n            self.float_status = P.FloatStatus()\n            self.addn = P.AddN()\n            self.reshape = P.Reshape()\n        else:\n            self.alloc_status = P.NPUAllocFloatStatus()\n            self.get_status = P.NPUGetFloatStatus()\n            self.clear_status = P.NPUClearFloatStatus()\n        self.reduce_sum = P.ReduceSum(keep_dims=False)\n        self.base = Tensor(1, mstype.float32)\n        self.less_equal = P.LessEqual()\n        self.hyper_map = C.HyperMap()\n        self.loss_scale = None\n        self.loss_scaling_manager = scale_update_cell\n        if scale_update_cell:\n            self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32))\n\n    def construct(self,\n                  input_ids,\n                  input_mask,\n                  token_type_id,\n                  label_ids,\n                  sens=None):\n        \"\"\"Bert Finetune\"\"\"\n        \n        weights = self.weights\n        init = False\n        loss = self.network(input_ids,\n                            input_mask,\n                            token_type_id,\n                            label_ids)\n        if sens is None:\n            scaling_sens = self.loss_scale\n        else:\n            scaling_sens = sens\n\n        if not self.gpu_target:\n            init = self.alloc_status()\n            init = F.depend(init, loss)\n            clear_status = self.clear_status(init)\n            scaling_sens = F.depend(scaling_sens, clear_status)\n        grads = self.grad(self.network, weights)(input_ids,\n                                                 input_mask,\n                                                 token_type_id,\n                                                 label_ids,\n                                                 self.cast(scaling_sens,\n                                                           mstype.float32))\n\n        self.optimizer(grads)\n        return loss\n",[207],{"type":18,"tag":131,"props":208,"children":209},{"__ignoreMap":7},[210],{"type":24,"value":205},{"type":18,"tag":44,"props":212,"children":214},{"id":213},"_3任务训练",[215],{"type":18,"tag":38,"props":216,"children":217},{},[218],{"type":24,"value":219},"3、任务训练",{"type":18,"tag":126,"props":221,"children":223},{"code":222},"from mindspore.train.callback import Callback\nfrom mindspore.train.callback import TimeMonitor\nfrom mindspore.train import Model\nfrom mindspore.nn.optim import AdamWeightDecay\nfrom mindspore.nn.wrap.loss_scale import DynamicLossScaleUpdateCell\nfrom mindspore import save_checkpoint, context, load_checkpoint, load_param_into_net\nfrom mindtext.modules.encoder.bert import BertConfig\nfrom bert import BertforSequenceClassification, BertLearningRate, BertFinetuneCell\nfrom bert_embedding import BertEmbedding\nimport LCQMCDataset\nfrom mindspore.common.tensor import Tensor\nimport time\n\n\ndef get_ms_timestamp():\n    t = time.time()\n    return int(round(t * 1000))\n\n\nclass LossCallBack(Callback):\n    \"\"\"\n    Monitor the loss in training.\n\n    If the loss is NAN or INF terminating training.\n\n    Note:\n        If per_print_times is 0 do not print loss.\n\n    Args:\n        per_print_times (int): Print loss every times. Default: 1.\n    \"\"\"\n\n    def __init__(self, per_print_times=1, rank_ids=0):\n        super(LossCallBack, self).__init__()\n        if not isinstance(per_print_times, int) or per_print_times \u003C 0:\n            raise ValueError(\"print_step must be int and >= 0.\")\n        self._per_print_times = per_print_times\n        self.rank_id = rank_ids\n        self.time_stamp_first = get_ms_timestamp()\n\n    def step_end(self, run_context):\n        \"\"\"Monitor the loss in training.\"\"\"\n        time_stamp_current = get_ms_timestamp()\n        cb_params = run_context.original_args()\n        print(\"time: {}, epoch: {}, step: {}, outputs are {}\".format(time_stamp_current - self.time_stamp_first,\n                                                                     cb_params.cur_epoch_num,\n                                                                     cb_params.cur_step_num,\n                                                                     str(cb_params.net_outputs)))\n        with open(\"./loss_{}.log\".format(self.rank_id), \"a+\") as f:\n            f.write(\"time: {}, epoch: {}, step: {}, loss: {}\".format(\n                time_stamp_current - self.time_stamp_first,\n                cb_params.cur_epoch_num,\n                cb_params.cur_step_num,\n                str(cb_params.net_outputs.asnumpy())))\n            f.write('\\n')\n\n\ndef train(train_data, bert, optimizer, save_path, epoch_num):\n    update_cell = DynamicLossScaleUpdateCell(loss_scale_value=2 ** 32, scale_factor=2, scale_window=1000)\n    netwithgrads = BertFinetuneCell(bert, optimizer=optimizer, scale_update_cell=update_cell)\n    callbacks = [TimeMonitor(train_data.get_dataset_size()), LossCallBack(train_data.get_dataset_size())]\n    model = Model(netwithgrads)\n    model.train(epoch_num, train_data, callbacks=callbacks, dataset_sink_mode=False)\n    save_checkpoint(model.train_network.network, save_path)\n\ndef main():\n    #context.set_context(mode=context.PYNATIVE_MODE, device_target=\"GPU\")\n    context.set_context(mode=0, device_target=\"GPU\")\n    #context.set_context(enable_graph_kernel=True)\n\n    epoch_num = 6\n    save_path = \"./model/output/train_lcqmc2.ckpt\"\n\n   \n    dataset = LCQMCDataset(paths='./dataset/lcqmc',\n                      tokenizer=\"./model\",\n                      max_length=128,\n                      truncation_strategy=True,\n                      batch_size=32, columns_list=['input_ids', 'attention_mask', 'token_type_ids', 'label'],\n                      test_columns_list=['input_ids', 'attention_mask', 'token_type_ids', 'label'])\n\n    ds = dataset.from_cache(batch_size=128,\n                      columns_list=['input_ids', 'attention_mask', 'token_type_ids', 'label'],\n                      test_columns_list=['input_ids', 'attention_mask', 'token_type_ids'])\n    train_data = ds['train']\n    bert_config_file = \"./model/test.yaml\"\n    bert_config = BertConfig.from_yaml_file(bert_config_file)\n    model_path = \"./model/bert_cn.ckpt\"\n    bert = BertforSequenceClassification(bert_config, True, num_labels=2, dropout_prob=0.1, multi_sample_dropout=5, label_smooth=0.9)\n    eb = BertEmbedding(bert_config, True)\n    eb.from_pretrain(model_path)\n    bert.init_embedding(eb)\n\n    lr_schedule = BertLearningRate(learning_rate=2e-5,\n                                   end_learning_rate=2e-5 * 0 ,\n                                   warmup_steps=int(train_data.get_dataset_size() * epoch_num * 0.1),\n                                   decay_steps=train_data.get_dataset_size() * epoch_num,\n                                   power=1.0)\n    params = bert.trainable_params()\n    optimizer = AdamWeightDecay(params, lr_schedule, eps=1e-8)\n\n    train(train_data, bert, optimizer, save_path, epoch_num)\n\n\nif __name__ == \"__main__\":\n    main()\n",[224],{"type":18,"tag":131,"props":225,"children":226},{"__ignoreMap":7},[227],{"type":24,"value":222},{"type":18,"tag":26,"props":229,"children":230},{},[231],{"type":24,"value":232},"关键参数：",{"type":18,"tag":26,"props":234,"children":235},{},[236],{"type":24,"value":237},"bert_config = BertConfig.from_yaml_file(bert_config_file)：读取Bert的配置参数",{"type":18,"tag":26,"props":239,"children":240},{},[241],{"type":24,"value":242},"eb.from_pretrain(model_path) ：加载Bert的MindSpore权重文件 bert.init_embedding(eb)：初始化加载的权重",{"type":18,"tag":26,"props":244,"children":245},{},[246],{"type":24,"value":247},"lr_schedule ：学习率控制器",{"type":18,"tag":26,"props":249,"children":250},{},[251],{"type":24,"value":252},"optimizer：梯度优化器",{"type":18,"tag":26,"props":254,"children":255},{},[256],{"type":18,"tag":30,"props":257,"children":259},{"alt":7,"src":258},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/11/09/c3428b3c43074bb8944d300860b79686.png",[],{"type":18,"tag":26,"props":261,"children":262},{},[263],{"type":24,"value":264},"使用lcaqmc的测试集来作为评估训练，输出模型在测试集中的精确度",{"type":18,"tag":126,"props":266,"children":268},{"code":267},"from mindspore.nn import Accuracy\nfrom tqdm import tqdm\nfrom mindspore import context\nimport BertforSequenceClassification\nimport BertConfig\nimport mindspore\nimport LCQMCDataset\n\n\ndef eval(eval_data, model):\n    metirc = Accuracy('classification')\n    metirc.clear()\n    squeeze = mindspore.ops.Squeeze(1)\n    for batch in tqdm(eval_data.create_dict_iterator(num_epochs=1), total=eval_data.get_dataset_size()):\n        input_ids = batch['input_ids']\n        token_type_id = batch['token_type_ids']\n        input_mask = batch['attention_mask']\n        label_ids = batch['label']\n        inputs = {\"input_ids\": input_ids,\n                  \"input_mask\": input_mask,\n                  \"token_type_id\": token_type_id\n                  }\n        output = model(**inputs)\n        sm = mindspore.nn.Softmax(axis=-1)\n        output = sm(output)\n        #print(output)\n        metirc.update(output, squeeze(label_ids))\n    print(metirc.eval())\n\n\ndef main():\n    context.set_context(mode=0, device_target=\"GPU\")\n\n    dataset = LCQMCDataset(paths='./dataset/lcqmc',\n                      tokenizer=\"./model\",\n                      max_length=128,\n                      truncation_strategy=True,\n                      batch_size=128, columns_list=['input_ids', 'attention_mask', 'token_type_ids', 'label'],\n                      test_columns_list=['input_ids', 'attention_mask', 'token_type_ids', 'label'])\n\n    #ds = dataset()\n    ds = dataset.from_cache(batch_size=128,\n                      columns_list=['input_ids', 'attention_mask', 'token_type_ids', 'label'],\n                      test_columns_list=['input_ids', 'attention_mask', 'token_type_ids','label'])\n\n    eval_data = ds['test']\n\n\n    bert_config_file = \"./model/test.yaml\"\n    bert_config = BertConfig.from_yaml_file(bert_config_file)\n\n    bert = BertforSequenceClassification(bert_config, is_training=False, num_labels=2,  dropout_prob=0.0)\n    model_path = \"./model/output/train_lcqmc2.ckpt\"\n    bert.from_pretrain(model_path)\n\n    eval(eval_data, bert)\n\nif __name__ == \"__main__\":\n    main()\n",[269],{"type":18,"tag":131,"props":270,"children":271},{"__ignoreMap":7},[272],{"type":24,"value":267},{"type":18,"tag":26,"props":274,"children":275},{},[276,280,282],{"type":18,"tag":30,"props":277,"children":279},{"alt":7,"src":278},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/11/09/d559aa55981b4e5fab4320e3dbacc737.png",[],{"type":24,"value":281}," ",{"type":18,"tag":30,"props":283,"children":285},{"alt":7,"src":284},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/11/09/24a2a4d07d9242d98e567d535c8c8aee.jpg",[],{"type":18,"tag":26,"props":287,"children":288},{},[289],{"type":24,"value":290},"模型在对应的数据集的验证集和验证集精确度",{"type":18,"tag":26,"props":292,"children":293},{},[294],{"type":18,"tag":30,"props":295,"children":297},{"alt":7,"src":296},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/11/09/0cddef403158450b930b5785bbbb144d.png",[],{"type":18,"tag":26,"props":299,"children":300},{},[301,303],{"type":24,"value":302},"**欢迎大家踊跃投稿，如果想投稿的同学****可以添加MindSpore官方小助手：****小猫子（mindspore0328）的微信，**",{"type":18,"tag":38,"props":304,"children":305},{},[306],{"type":24,"value":307},"告诉猫哥哦！",{"type":18,"tag":26,"props":309,"children":310},{},[311],{"type":24,"value":312},"扫描下方二维码加入MindSpore项目↓",{"type":18,"tag":26,"props":314,"children":315},{},[316],{"type":18,"tag":30,"props":317,"children":319},{"alt":7,"src":318},"https://obs-mindspore-file.obs.cn-north-4.myhuaweicloud.com/file/2021/11/09/390d369804c94e1693536b94b1c7d738.jpg",[],{"type":18,"tag":44,"props":321,"children":322},{"id":7},[],{"type":18,"tag":26,"props":324,"children":325},{},[326],{"type":24,"value":327},"MindSpore官方资料",{"type":18,"tag":26,"props":329,"children":330},{},[331,333],{"type":24,"value":332},"GitHub : ",{"type":18,"tag":334,"props":335,"children":339},"a",{"href":336,"rel":337},"https://github.com/mindspore-ai/mindspore",[338],"nofollow",[340],{"type":24,"value":336},{"type":18,"tag":26,"props":342,"children":343},{},[344],{"type":24,"value":345},"Gitee : https : //gitee.com/mindspore/mindspore",{"type":18,"tag":26,"props":347,"children":348},{},[349],{"type":24,"value":350},"官方QQ群 : 871543426",{"title":7,"searchDepth":352,"depth":352,"links":353},4,[354,356,357,358,359],{"id":46,"depth":355,"text":52},3,{"id":162,"depth":355,"text":168},{"id":184,"depth":355,"text":190},{"id":213,"depth":355,"text":219},{"id":7,"depth":355,"text":7},"markdown","content:technology-blogs:zh:780.md","content","technology-blogs/zh/780.md","technology-blogs/zh/780","md",1776506141152]