@ -0,0 +1,761 @@
{
"cells": [
{
"cell_type": "markdown",
"source": [
"## relation extraction experiment\n",
"> Tutorial author: 陶联宽(22051063@zju.edu.cn)\n",
"\n",
"On this demo, we use `lstm` to extract attributions.We hope this demo can help you understand the process of construction knowledge graph and the principles and common methods of triplet extraction.\n",
"\n",
"This demo uses `Python3`.\n",
"\n",
"### Dataset\n",
"In this example,we get some Chinese text to extract the triples\n",
"\n",
"sentence|attribute|entity|entity_offset|attribute_value|attribute_value_offset\n",
":---:|:---:|:---:|:---:|:---:|:---:\n",
"苏轼（ ～ ） ， ， ， ， （ ） ， （ ）
"阳成俊，男，汉族，贵州省委党校大学学历|民族|阳成俊|0|汉族|6\n",
"司马懿，字仲达，河南温县人|字|司马懿｜
"\n",
"- train.csv: It contains 6 training triples,each lines represent one triple,sorted by sentence,attribute,entity,entity's offset,attribute value attribute value's offset,and separated by ,.\n",
"- valid.csv: It contains 2 training triples,each lines represent one triple,sorted by sentence,attribute,entity,entity's offset,attribute value attribute value's offset,and separated by ,.\n",
"- test.csv: It contains 2 training triples,each lines represent one triple,sorted by sentence,attribute,entity,entity's offset,attribute value attribute value's offset,and separated by ,.\n",
"- attribute.csv: It contains 3 attribute triples,each lines sorted by attribute,index and separated by ,.\n"
],
"metadata": {}
},
{
"cell_type": "markdown",
"source": [
"### LSTM\n",
"\n",
"\n",
"\n",
"The sentence information mainly includes wording embedding.After the rnn layer,according to the position of entity,attribute key,it through the full connection layer, the attribution information of the sentence can be obtained."
],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# Run the neural network with pytorch and confirm whether it is installed before running\n",
"!pip install torch\n",
"!pip install matplotlib\n",
"!pip install transformers"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# import the whole modules\n",
"import os\n",
"import csv\n",
"import math\n",
"import pickle\n",
"import logging\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from torch import optim\n",
"from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
"from torch.utils.data import Dataset,DataLoader\n",
"from sklearn.metrics import precision_recall_fscore_support\n",
"from typing import List, Tuple, Dict, Any, Sequence, Optional, Union\n",
"from transformers import BertTokenizer, BertModel\n",
"\n",
"logger = logging.getLogger(__name__)"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# Configuration file of model parameters\n",
"# use_pcnn Parameter controls whether there is a piece_ Wise pooling\n",
"\n",
"class Config(object):\n",
" model_name = 'rnn' # ['cnn', 'gcn', 'lm','rnn']\n",
" use_pcnn = False\n",
" min_freq = 1\n",
" pos_limit = 20\n",
" out_path = 'data/out' \n",
" batch_size = 2 \n",
" word_dim = 10\n",
" pos_dim = 5\n",
" dim_strategy = 'sum' # ['sum', 'cat']\n",
" out_channels = 20\n",
" intermediate = 10\n",
" kernel_sizes = [3, 5, 7]\n",
" activation = 'gelu'\n",
" pooling_strategy = 'max'\n",
" dropout = 0.3\n",
" epoch = 10\n",
" num_relations = 4\n",
" learning_rate = 3e-4\n",
" lr_factor = 0.7 # 学习率的衰减率\n",
" lr_patience = 3 # 学习率衰减的等待epoch\n",
" weight_decay = 1e-3 # L2正则\n",
" early_stopping_patience = 6\n",
" train_log = True\n",
" log_interval = 1\n",
" show_plot = True\n",
" only_comparison_plot = False\n",
" plot_utils = 'matplot'\n",
" lm_file = 'bert-base-chinese'\n",
" lm_num_hidden_layers = 2\n",
" rnn_layers = 2\n",
" \n",
"cfg = Config()"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# Word token builds a one hot dictionary, and then inputs it to the embedding layer to obtain the corresponding word information matrix\n",
"# 0 is pad by default and 1 is unknown\n",
"class Vocab(object):\n",
" def __init__(self, name: str = 'basic', init_tokens = [\"[PAD]\", \"[UNK]\"]):\n",
" self.name = name\n",
" self.init_tokens = init_tokens\n",
" self.trimed = False\n",
" self.word2idx = {}\n",
" self.word2count = {}\n",
" self.idx2word = {}\n",
" self.count = 0\n",
" self._add_init_tokens()\n",
"\n",
" def _add_init_tokens(self):\n",
" for token in self.init_tokens:\n",
" self._add_word(token)\n",
"\n",
" def _add_word(self, word: str):\n",
" if word not in self.word2idx:\n",
" self.word2idx[word] = self.count\n",
" self.word2count[word] = 1\n",
" self.idx2word[self.count] = word\n",
" self.count += 1\n",
" else:\n",
" self.word2count[word] += 1\n",
"\n",
" def add_words(self, words: Sequence):\n",
" for word in words:\n",
" self._add_word(word)\n",
"\n",
" def trim(self, min_freq=2, verbose: Optional[bool] = True):\n",
" assert min_freq == int(min_freq), f'min_freq must be integer, can\\'t be {min_freq}'\n",
" min_freq = int(min_freq)\n",
" if min_freq < 2:\n",
" return\n",
" if self.trimed:\n",
" return\n",
" self.trimed = True\n",
"\n",
" keep_words = []\n",
" new_words = []\n",
"\n",
" for k, v in self.word2count.items():\n",
" if v >= min_freq:\n",
" keep_words.append(k)\n",
" new_words.extend([k] * v)\n",
" if verbose:\n",
" before_len = len(keep_words)\n",
" after_len = len(self.word2idx) - len(self.init_tokens)\n",
" logger.info('vocab after be trimmed, keep words [{} / {}] = {:.2f}%'.format(before_len, after_len, before_len / after_len * 100))\n",
"\n",
" # Reinitialize dictionaries\n",
" self.word2idx = {}\n",
" self.word2count = {}\n",
" self.idx2word = {}\n",
" self.count = 0\n",
" self._add_init_tokens()\n",
" self.add_words(new_words)"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# Functions required for preprocessing\n",
"Path = str\n",
"\n",
"def load_csv(fp: Path, is_tsv: bool = False, verbose: bool = True) -> List:\n",
" if verbose:\n",
" logger.info(f'load csv from {fp}')\n",
"\n",
" dialect = 'excel-tab' if is_tsv else 'excel'\n",
" with open(fp, encoding='utf-8') as f:\n",
" reader = csv.DictReader(f, dialect=dialect)\n",
" return list(reader)\n",
"\n",
" \n",
"def load_pkl(fp: Path, verbose: bool = True) -> Any:\n",
" if verbose:\n",
" logger.info(f'load data from {fp}')\n",
"\n",
" with open(fp, 'rb') as f:\n",
" data = pickle.load(f)\n",
" return data\n",
"\n",
"\n",
"def save_pkl(data: Any, fp: Path, verbose: bool = True) -> None:\n",
" if verbose:\n",
" logger.info(f'save data in {fp}')\n",
"\n",
" with open(fp, 'wb') as f:\n",
" pickle.dump(data, f)\n",
" \n",
"def _handle_attribute_data(attribute_data: List[Dict]) -> Dict:\n",
" atts = OrderedDict()\n",
" attribute_data = sorted(attribute_data, key=lambda i: int(i['index']))\n",
" for d in attribute_data:\n",
" atts[d['attribute']] = {\n",
" 'index': int(d['index'])\n",
" }\n",
" return atts\n",
"\n",
"def _add_attribute_data(atts: Dict, data: List) -> None:\n",
" for d in data:\n",
" d['att2idx'] = atts[d['attribute']]['index']\n",
"\n",
"def _convert_tokens_into_index(data: List[Dict], vocab):\n",
" unk_str = '[UNK]'\n",
" unk_idx = vocab.word2idx[unk_str]\n",
"\n",
" for d in data:\n",
" d['token2idx'] = [vocab.word2idx.get(i, unk_idx) for i in d['tokens']]\n",
"\n",
"def _add_pos_seq(train_data: List[Dict], cfg):\n",
" for d in train_data:\n",
" d['entity_pos'] = list(map(lambda i: i - d['entity_index'], list(range(d['seq_len']))))\n",
" d['entity_pos'] = _handle_pos_limit(d['entity_pos'],int(cfg.pos_limit))\n",
"\n",
" d['attribute_value_pos'] = list(map(lambda i: i - d['attribute_value_index'], list(range(d['seq_len']))))\n",
" d['attribute_value_pos'] = _handle_pos_limit(d['attribute_value_pos'],int(cfg.pos_limit))\n",
" \n",
"def _handle_pos_limit(pos: List[int], limit: int) -> List[int]:\n",
" for i,p in enumerate(pos):\n",
" if p > limit:\n",
" pos[i] = limit\n",
" if p < -limit:\n",
" pos[i] = -limit\n",
" return [p + limit + 1 for p in pos]\n",
"\n",
"def seq_len_to_mask(seq_len: Union[List, np.ndarray, torch.Tensor], max_len=None, mask_pos_to_true=True):\n",
" if isinstance(seq_len, list):\n",
" seq_len = np.array(seq_len)\n",
"\n",
" if isinstance(seq_len, np.ndarray):\n",
" seq_len = torch.from_numpy(seq_len)\n",
"\n",
" if isinstance(seq_len, torch.Tensor):\n",
" assert seq_len.dim() == 1, logger.error(f\"seq_len can only have one dimension, got {seq_len.dim()} != 1.\")\n",
" batch_size = seq_len.size(0)\n",
" max_len = int(max_len) if max_len else seq_len.max().long()\n",
" broad_cast_seq_len = torch.arange(max_len).expand(batch_size, -1).to(seq_len.device)\n",
" if mask_pos_to_true:\n",
" mask = broad_cast_seq_len.ge(seq_len.unsqueeze(1))\n",
" else:\n",
" mask = broad_cast_seq_len.lt(seq_len.unsqueeze(1))\n",
" else:\n",
" raise logger.error(\"Only support 1-d list or 1-d numpy.ndarray or 1-d torch.Tensor.\")\n",
"\n",
" return mask\n"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# Preprocess\n",
"logger.info('load raw files...')\n",
"train_fp = os.path.join('data/train.csv')\n",
"valid_fp = os.path.join('data/valid.csv')\n",
"test_fp = os.path.join('data/test.csv')\n",
"attribute_fp = os.path.join('data/attribute.csv')\n",
"\n",
"train_data = load_csv(train_fp)\n",
"valid_data = load_csv(valid_fp)\n",
"test_data = load_csv(test_fp)\n",
"attribute_data = load_csv(attribute_fp)\n",
"\n",
"for d in train_data:\n",
" d['tokens'] = eval(d['tokens'])\n",
"for d in valid_data:\n",
" d['tokens'] = eval(d['tokens'])\n",
"for d in test_data:\n",
" d['tokens'] = eval(d['tokens'])\n",
"\n",
"logger.info('convert relation into index...')\n",
"atts = _handle_attribute_data(attribute_data)\n",
"_add_attribute_data(atts,train_data)\n",
"_add_attribute_data(atts,test_data)\n",
"_add_attribute_data(atts,valid_data)\n",
"\n",
"logger.info('build vocabulary...')\n",
"vocab = Vocab('word')\n",
"train_tokens = [d['tokens'] for d in train_data]\n",
"valid_tokens = [d['tokens'] for d in valid_data]\n",
"test_tokens = [d['tokens'] for d in test_data]\n",
"sent_tokens = [*train_tokens, *valid_tokens, *test_tokens]\n",
"for sent in sent_tokens:\n",
" vocab.add_words(sent)\n",
"vocab.trim(min_freq=cfg.min_freq)\n",
"\n",
"logger.info('convert tokens into index...')\n",
"_convert_tokens_into_index(train_data, vocab)\n",
"_convert_tokens_into_index(valid_data, vocab)\n",
"_convert_tokens_into_index(test_data, vocab)\n",
"\n",
"logger.info('build position sequence...')\n",
"_add_pos_seq(train_data, cfg)\n",
"_add_pos_seq(valid_data, cfg)\n",
"_add_pos_seq(test_data, cfg)\n",
"\n",
"logger.info('save data for backup...')\n",
"os.makedirs(cfg.out_path, exist_ok=True)\n",
"train_save_fp = os.path.join(cfg.out_path, 'train.pkl')\n",
"valid_save_fp = os.path.join(cfg.out_path, 'valid.pkl')\n",
"test_save_fp = os.path.join(cfg.out_path, 'test.pkl')\n",
"save_pkl(train_data, train_save_fp)\n",
"save_pkl(valid_data, valid_save_fp)\n",
"save_pkl(test_data, test_save_fp)\n",
"\n",
"vocab_save_fp = os.path.join(cfg.out_path, 'vocab.pkl')\n",
"vocab_txt = os.path.join(cfg.out_path, 'vocab.txt')\n",
"save_pkl(vocab, vocab_save_fp)\n",
"logger.info('save vocab in txt file, for watching...')\n",
"with open(vocab_txt, 'w', encoding='utf-8') as f:\n",
" f.write(os.linesep.join(vocab.word2idx.keys()))\n",
"\n"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# pytorch construct Dataset\n",
"def collate_fn(cfg):\n",
" def collate_fn_intra(batch):\n",
" batch.sort(key=lambda data: data['seq_len'], reverse=True)\n",
" max_len = batch[0]['seq_len']\n",
"\n",
" def _padding(x, max_len):\n",
" return x + [0] * (max_len - len(x))\n",
"\n",
" x, y = dict(), []\n",
" word, word_len = [], []\n",
" head_pos, tail_pos = [], []\n",
" pcnn_mask = []\n",
" for data in batch:\n",
" word.append(_padding(data['token2idx'], max_len))\n",
" word_len.append(data['seq_len'])\n",
" y.append(int(data['att2idx']))\n",
"\n",
" if cfg.model_name != 'lm':\n",
" head_pos.append(_padding(data['entity_pos'], max_len))\n",
" tail_pos.append(_padding(data['attribute_value_pos'], max_len))\n",
" if cfg.model_name == 'cnn':\n",
" if cfg.use_pcnn:\n",
" pcnn_mask.append(_padding(data['entities_pos'], max_len))\n",
"\n",
" x['word'] = torch.tensor(word)\n",
" x['lens'] = torch.tensor(word_len)\n",
" y = torch.tensor(y)\n",
"\n",
" if cfg.model_name != 'lm':\n",
" x['entity_pos'] = torch.tensor(head_pos)\n",
" x['attribute_value_pos'] = torch.tensor(tail_pos)\n",
" if cfg.model_name == 'cnn' and cfg.use_pcnn:\n",
" x['pcnn_mask'] = torch.tensor(pcnn_mask)\n",
" if cfg.model_name == 'gcn':\n",
" # 没找到合适的做 parsing tree 的工具，暂时随机初始化\n",
" B, L = len(batch), max_len\n",
" adj = torch.empty(B, L, L).random_(2)\n",
" x['adj'] = adj\n",
" return x, y\n",
"\n",
" return collate_fn_intra\n",
"\n",
"\n",
"class CustomDataset(Dataset):\n",
" \"\"\"\n",
" 默认使用 List 存储数据\n",
" \"\"\"\n",
" def __init__(self, fp):\n",
" self.file = load_pkl(fp)\n",
"\n",
" def __getitem__(self, item):\n",
" sample = self.file[item]\n",
" return sample\n",
"\n",
" def __len__(self):\n",
" return len(self.file)"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# embedding layer\n",
"import torch\n",
"import torch.nn as nn\n",
"\n",
"\n",
"class Embedding(nn.Module):\n",
" def __init__(self, config):\n",
" \"\"\"\n",
" word embedding: 一般 0 为 padding\n",
" pos embedding: 一般 0 为 padding\n",
" dim_strategy: [cat, sum] 多个 embedding 是拼接还是相加\n",
" \"\"\"\n",
" super(Embedding, self).__init__()\n",
"\n",
" # self.xxx = config.xxx\n",
" self.vocab_size = config.vocab_size\n",
" self.word_dim = config.word_dim\n",
" self.pos_size = config.pos_size\n",
" self.pos_dim = config.pos_dim if config.dim_strategy == 'cat' else config.word_dim\n",
" self.dim_strategy = config.dim_strategy\n",
"\n",
" self.wordEmbed = nn.Embedding(self.vocab_size, self.word_dim, padding_idx=0)\n",
" self.entityPosEmbed = nn.Embedding(self.pos_size, self.pos_dim, padding_idx=0)\n",
" self.attribute_keyPosEmbed = nn.Embedding(self.pos_size, self.pos_dim, padding_idx=0)\n",
" \n",
" self.layer_norm = nn.LayerNorm(self.word_dim)\n",
"\n",
" def forward(self, *x):\n",
" word, entity, attribute_key = x\n",
" word_embedding = self.wordEmbed(word)\n",
" entity_embedding = self.entityPosEmbed(head)\n",
" attribute_key_embedding = self.attribute_keyPosEmbed(tail)\n",
"\n",
" if self.dim_strategy == 'cat':\n",
" return torch.cat((word_embedding, entity_embedding, attribute_key_embedding), -1)\n",
" elif self.dim_strategy == 'sum':\n",
" # 此时 pos_dim == word_dim\n",
" return self.layer_norm(word_embedding + entity_embedding + attribute_key_embedding)\n",
" else:\n",
" raise Exception('dim_strategy must choose from [sum, cat]')\n"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# Gelu activation function, specified by transformer, works better than relu\n",
"class GELU(nn.Module):\n",
" def __init__(self):\n",
" super(GELU, self).__init__()\n",
"\n",
" def forward(self, x):\n",
" return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"#rnn model\n",
"class RNN(nn.Module):\n",
" def __init__(self, config):\n",
" \"\"\"\n",
" type_rnn: RNN, GRU, LSTM 可选\n",
" \"\"\"\n",
" super(RNN, self).__init__()\n",
"\n",
" # self.xxx = config.xxx\n",
" self.input_size = config.input_size\n",
" self.hidden_size = config.hidden_size // 2 if config.bidirectional else config.hidden_size\n",
" self.num_layers = config.num_layers\n",
" self.dropout = config.dropout\n",
" self.bidirectional = config.bidirectional\n",
" self.last_layer_hn = config.last_layer_hn\n",
" self.type_rnn = config.type_rnn\n",
"\n",
" rnn = eval(f'nn.{self.type_rnn}')\n",
" self.rnn = rnn(input_size=self.input_size,\n",
" hidden_size=self.hidden_size,\n",
" num_layers=self.num_layers,\n",
" dropout=self.dropout,\n",
" bidirectional=self.bidirectional,\n",
" bias=True,\n",
" batch_first=True)\n",
"\n",
" # 有bug\n",
" # self._init_weights()\n",
"\n",
" def _init_weights(self):\n",
" \"\"\"orthogonal init yields generally good results than uniform init\"\"\"\n",
" gain = 1 # use default value\n",
" for nth in range(self.num_layers * self.bidirectional):\n",
" # w_ih, (4 * hidden_size x input_size)\n",
" nn.init.orthogonal_(self.rnn.all_weights[nth][0], gain=gain)\n",
" # w_hh, (4 * hidden_size x hidden_size)\n",
" nn.init.orthogonal_(self.rnn.all_weights[nth][1], gain=gain)\n",
" # b_ih, (4 * hidden_size)\n",
" nn.init.zeros_(self.rnn.all_weights[nth][2])\n",
" # b_hh, (4 * hidden_size)\n",
" nn.init.zeros_(self.rnn.all_weights[nth][3])\n",
"\n",
" def forward(self, x, x_len):\n",
" \"\"\"\n",
" Args: \n",
" torch.Tensor [batch_size, seq_max_length, input_size], [B, L, H_in] 一般是经过embedding后的值\n",
" x_len: torch.Tensor [L] 已经排好序的句长值\n",
" Returns:\n",
" output: torch.Tensor [B, L, H_out] 序列标注的使用结果\n",
" hn: torch.Tensor [B, N, H_out] / [B, H_out] 分类的结果，当 last_layer_hn 时只有最后一层结果\n",
" \"\"\"\n",
" B, L, _ = x.size()\n",
" H, N = self.hidden_size, self.num_layers\n",
"\n",
" x_len = x_len.cpu()\n",
" x = pack_padded_sequence(x, x_len, batch_first=True, enforce_sorted=True)\n",
" output, hn = self.rnn(x)\n",
" output, _ = pad_packed_sequence(output, batch_first=True, total_length=L)\n",
"\n",
" if self.type_rnn == 'LSTM':\n",
" hn = hn[0]\n",
" if self.bidirectional:\n",
" hn = hn.view(N, 2, B, H).transpose(1, 2).contiguous().view(N, B, 2 * H).transpose(0, 1)\n",
" else:\n",
" hn = hn.transpose(0, 1)\n",
" if self.last_layer_hn:\n",
" hn = hn[:, -1, :]\n",
"\n",
" return output, hn"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# p,r,f1 measurement\n",
"class PRMetric():\n",
" def __init__(self):\n",
" \n",
" self.y_true = np.empty(0)\n",
" self.y_pred = np.empty(0)\n",
"\n",
" def reset(self):\n",
" self.y_true = np.empty(0)\n",
" self.y_pred = np.empty(0)\n",
"\n",
" def update(self, y_true:torch.Tensor, y_pred:torch.Tensor):\n",
" y_true = y_true.cpu().detach().numpy()\n",
" y_pred = y_pred.cpu().detach().numpy()\n",
" y_pred = np.argmax(y_pred,axis=-1)\n",
"\n",
" self.y_true = np.append(self.y_true, y_true)\n",
" self.y_pred = np.append(self.y_pred, y_pred)\n",
"\n",
" def compute(self):\n",
" p, r, f1, _ = precision_recall_fscore_support(self.y_true,self.y_pred,average='macro',warn_for=tuple())\n",
" _, _, acc, _ = precision_recall_fscore_support(self.y_true,self.y_pred,average='micro',warn_for=tuple())\n",
"\n",
" return acc,p,r,f1"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# Iteration in training process\n",
"def train(epoch, model, dataloader, optimizer, criterion, cfg):\n",
" model.train()\n",
"\n",
" metric = PRMetric()\n",
" losses = []\n",
"\n",
" for batch_idx, (x, y) in enumerate(dataloader, 1):\n",
" optimizer.zero_grad()\n",
" y_pred = model(x)\n",
" loss = criterion(y_pred, y)\n",
"\n",
" loss.backward()\n",
" optimizer.step()\n",
"\n",
" metric.update(y_true=y, y_pred=y_pred)\n",
" losses.append(loss.item())\n",
"\n",
" data_total = len(dataloader.dataset)\n",
" data_cal = data_total if batch_idx == len(dataloader) else batch_idx * len(y)\n",
" if (cfg.train_log and batch_idx % cfg.log_interval == 0) or batch_idx == len(dataloader):\n",
" acc,p,r,f1 = metric.compute()\n",
" print(f'Train Epoch {epoch}: [{data_cal}/{data_total} ({100. * data_cal / data_total:.0f}%)]\\t'\n",
" f'Loss: {loss.item():.6f}')\n",
" print(f'Train Epoch {epoch}: Acc: {100. * acc:.2f}%\\t'\n",
" f'macro metrics: [p: {p:.4f}, r:{r:.4f}, f1:{f1:.4f}]')\n",
"\n",
" if cfg.show_plot and not cfg.only_comparison_plot:\n",
" if cfg.plot_utils == 'matplot':\n",
" plt.plot(losses)\n",
" plt.title(f'epoch {epoch} train loss')\n",
" plt.show()\n",
"\n",
" return losses[-1]\n",
"\n",
"# Iteration in testing process\n",
"def validate(epoch, model, dataloader, criterion,verbose=True):\n",
" model.eval()\n",
"\n",
" metric = PRMetric()\n",
" losses = []\n",
"\n",
" for batch_idx, (x, y) in enumerate(dataloader, 1):\n",
" with torch.no_grad():\n",
" y_pred = model(x)\n",
" loss = criterion(y_pred, y)\n",
"\n",
" metric.update(y_true=y, y_pred=y_pred)\n",
" losses.append(loss.item())\n",
"\n",
" loss = sum(losses) / len(losses)\n",
" acc,p,r,f1 = metric.compute()\n",
" data_total = len(dataloader.dataset)\n",
" if verbose:\n",
" print(f'Valid Epoch {epoch}: [{data_total}/{data_total}](100%)\\t Loss: {loss:.6f}')\n",
" print(f'Valid Epoch {epoch}: Acc: {100. * acc:.2f}%\\tmacro metrics: [p: {p:.4f}, r:{r:.4f}, f1:{f1:.4f}]\\n\\n')\n",
"\n",
" return f1,loss"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# Load dataset\n",
"train_dataset = CustomDataset(train_save_fp)\n",
"valid_dataset = CustomDataset(valid_save_fp)\n",
"test_dataset = CustomDataset(test_save_fp)\n",
"\n",
"train_dataloader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True, collate_fn=collate_fn(cfg))\n",
"valid_dataloader = DataLoader(valid_dataset, batch_size=cfg.batch_size, shuffle=True, collate_fn=collate_fn(cfg))\n",
"test_dataloader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=True, collate_fn=collate_fn(cfg))"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# After the preprocessed data is loaded, vocab_size is known\n",
"vocab = load_pkl(vocab_save_fp)\n",
"vocab_size = vocab.count\n",
"cfg.vocab_size = vocab_size"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "code",
"execution_count": null,
"source": [
"# main entry, define optimization function, loss function and so on\n",
"# start epoch\n",
"# Use the loss of the valid dataset to make an early stop judgment. When it does not decline, this is the time when the model generalization is the best.\n",
"model = RNN(cfg)\n",
"print(model)\n",
"\n",
"optimizer = optim.Adam(model.parameters(), lr=cfg.learning_rate, weight_decay=cfg.weight_decay)\n",
"scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=cfg.lr_factor, patience=cfg.lr_patience)\n",
"criterion = nn.CrossEntropyLoss()\n",
"\n",
"best_f1, best_epoch = -1, 0\n",
"es_loss, es_f1, es_epoch, es_patience, best_es_epoch, best_es_f1, = 1000, -1, 0, 0, 0, -1\n",
"train_losses, valid_losses = [], []\n",
"\n",
"logger.info('=' * 10 + ' Start training ' + '=' * 10)\n",
"for epoch in range(1, cfg.epoch + 1):\n",
" train_loss = train(epoch, model, train_dataloader, optimizer, criterion, cfg)\n",
" valid_f1, valid_loss = validate(epoch, model, valid_dataloader, criterion)\n",
" scheduler.step(valid_loss)\n",
"\n",
" train_losses.append(train_loss)\n",
" valid_losses.append(valid_loss)\n",
" if best_f1 < valid_f1:\n",
" best_f1 = valid_f1\n",
" best_epoch = epoch\n",
" # 使用 valid loss 做 early stopping 的判断标准\n",
" if es_loss > valid_loss:\n",
" es_loss = valid_loss\n",
" es_f1 = valid_f1\n",
" best_es_f1 = valid_f1\n",
" es_epoch = epoch\n",
" best_es_epoch = epoch\n",
" es_patience = 0\n",
" else:\n",
" es_patience += 1\n",
" if es_patience >= cfg.early_stopping_patience:\n",
" best_es_epoch = es_epoch\n",
" best_es_f1 = es_f1\n",
"\n",
"if cfg.show_plot:\n",
" if cfg.plot_utils == 'matplot':\n",
" plt.plot(train_losses, 'x-')\n",
" plt.plot(valid_losses, '+-')\n",
" plt.legend(['train', 'valid'])\n",
" plt.title('train/valid comparison loss')\n",
" plt.show()\n",
"\n",
"\n",
"print(f'best(valid loss quota) early stopping epoch: {best_es_epoch}, '\n",
" f'this epoch macro f1: {best_es_f1:0.4f}')\n",
"print(f'total {cfg.epoch} epochs, best(valid macro f1) epoch: {best_epoch}, '\n",
" f'this epoch macro f1: {best_f1:.4f}')\n",
"\n",
"test_f1, _ = validate(0, model, test_dataloader, criterion,verbose=False)\n",
"print(f'after {cfg.epoch} epochs, final test data macro f1: {test_f1:.4f}')\n"
],
"outputs": [],
"metadata": {}
},
{
"cell_type": "markdown",
"source": [
"This demo does not include parameter adjustment. Interested students can go to [deepke] by themselves（ ）
],
"metadata": {}
}
],
"metadata": {
"orig_nbformat": 4,
"language_info": {
"name": "plaintext"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
tlk-dsg
