2_BiLSTM_Att_SemEval2010.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CGkexf5l7cP3"
   },
   "source": [
    "# Attention et BiLSTM pour l'extraction de relation\n",
    "\n",
    "Dans ce notebook nous allons implémenter un modèle BiLSTM auquel s'ajoute le mécanisme d'attention. Ce modèle est inspiré de l'article [Attention-Based Bidirectional Long Short-Term Memory Networks for Relation Classification.](https://www.aclweb.org/anthology/P16-2034.pdf).\n",
    "\n",
    "## Préparation des données\n",
    "\n",
    "### Import des données \n",
    "\n",
    "On charge les données et on les converti au format JSON."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "Nd37Hd347oPV"
   },
   "outputs": [],
   "source": [
    "my_path = \"data_RE\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "obk-Ikql7cQC"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "\n",
    "def search_entity(sentence):\n",
    "    e1 = re.findall(r'<e1>(.*)</e1>', sentence)[0]\n",
    "    e2 = re.findall(r'<e2>(.*)</e2>', sentence)[0]\n",
    "    sentence = sentence.replace('<e1>' + e1 + '</e1>', ' <e1> ' + e1 + ' </e1> ', 1)\n",
    "    sentence = sentence.replace('<e2>' + e2 + '</e2>', ' <e2> ' + e2 + ' </e2> ', 1)\n",
    "    sentence = sentence.split()\n",
    "    sentence = ' '.join(sentence)\n",
    "    sentence = sentence.replace('< e1 >', '<e1>')\n",
    "    sentence = sentence.replace('< e2 >', '<e2>')\n",
    "    sentence = sentence.replace('< /e1 >', '</e1>')\n",
    "    sentence = sentence.replace('< /e2 >', '</e2>')\n",
    "    sentence = sentence.split()\n",
    "\n",
    "    assert '<e1>' in sentence\n",
    "    assert '<e2>' in sentence\n",
    "    assert '</e1>' in sentence\n",
    "    assert '</e2>' in sentence\n",
    "\n",
    "    return sentence\n",
    "\n",
    "\n",
    "def convert(path_src, path_des):\n",
    "    with open(path_src, 'r', encoding='utf-8') as fr:\n",
    "        data = fr.readlines()\n",
    "        \n",
    "    with open(path_des, 'w', encoding='utf-8') as fw:\n",
    "        for i in range(0, len(data), 4):\n",
    "            id_s, sentence = data[i].strip().split('\\t')\n",
    "            sentence = sentence[1:-1]\n",
    "            sentence = search_entity(sentence)\n",
    "            meta = dict(\n",
    "                id=id_s,\n",
    "                relation=data[i+1].strip(),\n",
    "                sentence=sentence,\n",
    "                comment=data[i+2].strip()[8:]\n",
    "            )\n",
    "            json.dump(meta, fw, ensure_ascii=False)\n",
    "            fw.write('\\n')\n",
    "\n",
    "\n",
    "path_train = my_path + '/TRAIN_FILE.TXT'\n",
    "path_test = my_path +'/TEST_FILE_FULL.TXT'\n",
    "path_valid = my_path +'/TEST_FILE_FULL.TXT'\n",
    "\n",
    "convert(path_train, my_path + '/train1.json')\n",
    "convert(path_test, my_path +'/test1.json')\n",
    "convert(path_test, my_path +'/valid1.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CQhqqIrp7cQv"
   },
   "source": [
    "### Import des embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "x7dwWJ5H7cQ3"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import torch\n",
    "import numpy as np\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "\n",
    "def load_embedding(path_word = my_path +'/embeddings/glove.6B.300d.txt' , word_dim = 300):\n",
    "    word2id = dict()  \n",
    "    word_vec = list() \n",
    "\n",
    "    word2id['PAD'] = len(word2id)   \n",
    "    word2id['UNK'] = len(word2id)  \n",
    "    word2id['<e1>'] = len(word2id)\n",
    "    word2id['<e2>'] = len(word2id)\n",
    "    word2id['</e1>'] = len(word2id)\n",
    "    word2id['</e2>'] = len(word2id)\n",
    "\n",
    "    with open(path_word, 'r', encoding='utf-8') as fr:\n",
    "        for line in fr:\n",
    "            line = line.strip().split()\n",
    "            if len(line) != word_dim + 1:\n",
    "                continue\n",
    "            word2id[line[0]] = len(word2id)\n",
    "            word_vec.append(np.asarray(line[1:], dtype=np.float32))\n",
    "\n",
    "    word_vec = np.stack(word_vec)\n",
    "    vec_mean, vec_std = word_vec.mean(), word_vec.std()\n",
    "    special_emb = np.random.normal(vec_mean, vec_std, (6, word_dim))\n",
    "    special_emb[0] = 0  # <pad> est initialisé à zéro\n",
    "\n",
    "    word_vec = np.concatenate((special_emb, word_vec), axis=0)\n",
    "    word_vec = word_vec.astype(np.float32).reshape(-1, word_dim)\n",
    "    word_vec = torch.from_numpy(word_vec)\n",
    "    return word2id, word_vec\n",
    "\n",
    "word2id, word_vec = load_embedding(path_word = my_path +'/embeddings/glove.6B.300d.txt' , word_dim = 300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DIZG1WIl7cSF"
   },
   "source": [
    "### Import des relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "oLIJsZJF7cSN"
   },
   "outputs": [],
   "source": [
    "def load_relation(data_dir = my_path):\n",
    "    relation_file = os.path.join(data_dir, 'relation2id.txt')\n",
    "    rel2id = {}\n",
    "    id2rel = {}\n",
    "    with open(relation_file, 'r', encoding='utf-8') as fr:\n",
    "        for line in fr:\n",
    "            if line.strip().split():\n",
    "    \n",
    "              relation, id_s = line.strip().split()\n",
    "              id_d = int(id_s)\n",
    "              rel2id[relation] = id_d\n",
    "              id2rel[id_d] = relation\n",
    "    return rel2id, id2rel, len(rel2id)\n",
    "\n",
    "rel2id, id2rel, class_num = load_relation()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "S7xSyaRG7cSv"
   },
   "source": [
    "### Traitement des données et itérateurs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "rf1qmne97cS0"
   },
   "outputs": [],
   "source": [
    "class BrevetsDateset(Dataset):\n",
    "    def __init__(self, filename, rel2id, word2id, max_len):\n",
    "        self.filename = filename\n",
    "        self.rel2id = rel2id\n",
    "        self.word2id = word2id\n",
    "        self.max_len = max_len\n",
    "        self.data_dir = my_path\n",
    "        self.dataset, self.label = self.__load_data()\n",
    "\n",
    "    def __symbolize_sentence(self, sentence):\n",
    "      \n",
    "        mask = [1] * len(sentence)\n",
    "        words = []\n",
    "        length = min(self.max_len, len(sentence))\n",
    "        mask = mask[:length]\n",
    "\n",
    "        for i in range(length):\n",
    "            words.append(self.word2id.get(sentence[i].lower(), self.word2id['UNK']))\n",
    "\n",
    "        if length < self.max_len:\n",
    "            for i in range(length, self.max_len):\n",
    "                mask.append(0)  # 'PAD' mask is zero\n",
    "                words.append(self.word2id['PAD'])\n",
    "\n",
    "        unit = np.asarray([words, mask], dtype=np.int64)\n",
    "        unit = np.reshape(unit, newshape=(1, 2, self.max_len))\n",
    "        return unit\n",
    "\n",
    "    def __load_data(self):\n",
    "        path_data_file = os.path.join(self.data_dir, self.filename)\n",
    "        data = []\n",
    "        labels = []\n",
    "        with open(path_data_file, 'r', encoding='utf-8') as fr:\n",
    "            for line in fr:\n",
    "                line = json.loads(line.strip())\n",
    "                label = line['relation']\n",
    "                sentence = line['sentence']\n",
    "                label_idx = self.rel2id[label]\n",
    "\n",
    "                one_sentence = self.__symbolize_sentence(sentence)\n",
    "                data.append(one_sentence)\n",
    "                labels.append(label_idx)\n",
    "        return data, labels\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        data = self.dataset[index]\n",
    "        label = self.label[index]\n",
    "        return data, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "OWXe1T4r7cTR"
   },
   "outputs": [],
   "source": [
    "class BrevetsDataLoader(object):\n",
    "    def __init__(self, rel2id, word2id, batch_size, max_len):\n",
    "        self.rel2id = rel2id\n",
    "        self.word2id = word2id\n",
    "        self.batch_size = batch_size\n",
    "        self.max_len = max_len\n",
    "\n",
    "    def __collate_fn(self, batch):\n",
    "        data, label = zip(*batch)  # unzip the batch data\n",
    "        data = list(data)\n",
    "        label = list(label)\n",
    "        data = torch.from_numpy(np.concatenate(data, axis=0))\n",
    "        label = torch.from_numpy(np.asarray(label, dtype=np.int64))\n",
    "        return data, label\n",
    "\n",
    "    def __get_data(self, filename,  shuffle=False):\n",
    "        dataset = BrevetsDateset(filename, self.rel2id, self.word2id, self.max_len)\n",
    "        loader = DataLoader(\n",
    "            dataset=dataset,\n",
    "            batch_size= self.batch_size,\n",
    "            shuffle=shuffle,\n",
    "            num_workers=2,\n",
    "            collate_fn=self.__collate_fn\n",
    "        )\n",
    "        return loader\n",
    "\n",
    "    def get_train(self):\n",
    "        return self.__get_data('train1.json', shuffle=True)\n",
    "\n",
    "    def get_dev(self):\n",
    "        return self.__get_data('valid1.json', shuffle=False)\n",
    "\n",
    "    def get_test(self):\n",
    "        return self.__get_data('test1.json', shuffle=False)\n",
    "\n",
    "batch_size = 10\n",
    "max_len = 500\n",
    "loader = BrevetsDataLoader(rel2id, word2id, batch_size, max_len)\n",
    "\n",
    "test_loader = loader.get_test()\n",
    "train_loader = loader.get_train()\n",
    "valid_loader = loader.get_dev()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mbIWCzWf7cT1"
   },
   "source": [
    "## Modèle\n",
    "\n",
    " - Architecture générale : \n",
    " \n",
    "![alt text](images/attbilstm.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "fXr8gM4e7cT9"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn import init\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "\n",
    "\n",
    "class Att_BLSTM(nn.Module):\n",
    "    def __init__(self, word_vec, class_num, max_len, word_dim, hidden_size, layers_num, dropout):\n",
    "        super().__init__()\n",
    "        self.word_vec = word_vec\n",
    "        self.class_num = class_num\n",
    "\n",
    "        self.max_len = max_len\n",
    "        self.word_dim = word_dim\n",
    "        self.hidden_size = hidden_size\n",
    "        self.layers_num = layers_num\n",
    "        self.dropout_value = dropout\n",
    "\n",
    "        self.word_embedding = nn.Embedding.from_pretrained(\n",
    "            embeddings=self.word_vec,\n",
    "            freeze=False,\n",
    "        )\n",
    "        self.lstm = nn.LSTM(\n",
    "            input_size=self.word_dim,\n",
    "            hidden_size=self.hidden_size,\n",
    "            num_layers=self.layers_num,\n",
    "            bias=True,\n",
    "            batch_first=True,\n",
    "            dropout=self.dropout_value,\n",
    "            bidirectional=True,\n",
    "        )\n",
    "        self.tanh = nn.Tanh()\n",
    "        self.dropout = nn.Dropout(self.dropout_value)\n",
    "\n",
    "        self.att_weight = nn.Parameter(torch.randn(1, self.hidden_size, 1))\n",
    "        self.dense = nn.Linear(\n",
    "            in_features=self.hidden_size,\n",
    "            out_features=self.class_num,\n",
    "            bias=True\n",
    "        )\n",
    "        init.xavier_normal_(self.dense.weight)\n",
    "        init.constant_(self.dense.bias, 0.)\n",
    "\n",
    "    def lstm_layer(self, x, mask):\n",
    "        lengths = torch.sum(mask.gt(0), dim=-1)\n",
    "        x = pack_padded_sequence(x, lengths, batch_first=True, enforce_sorted=False)\n",
    "        h, (_, _) = self.lstm(x)\n",
    "        h, _ = pad_packed_sequence(h, batch_first=True, padding_value=0.0, total_length=self.max_len)\n",
    "        h = h.view(-1, self.max_len, 2, self.hidden_size)\n",
    "        h = torch.sum(h, dim=2)  \n",
    "        return h\n",
    "\n",
    "    def attention_layer(self, h, mask):\n",
    "        att_weight = self.att_weight.expand(mask.shape[0], -1, -1) \n",
    "        att_score = torch.bmm(self.tanh(h), att_weight)  \n",
    "        mask = mask.unsqueeze(dim=-1)  \n",
    "        att_score = att_score.masked_fill(mask.eq(0), float('-inf')) \n",
    "        att_weight = F.softmax(att_score, dim=1) \n",
    "        reps = torch.bmm(h.transpose(1, 2), att_weight).squeeze(dim=-1) \n",
    "        reps = self.tanh(reps)  \n",
    "        return reps\n",
    "\n",
    "    def forward(self, data):\n",
    "        token = data[:, 0, :].contiguous().view(-1, self.max_len)\n",
    "        mask = data[:, 1, :].contiguous().view(-1, self.max_len)\n",
    "        emb = self.word_embedding(token) \n",
    "        emb = self.dropout(emb)\n",
    "        h = self.lstm_layer(emb, mask) \n",
    "        h = self.dropout(h)\n",
    "        reps = self.attention_layer(h, mask) \n",
    "        reps = self.dropout(reps)\n",
    "        logits = self.dense(reps)\n",
    "        return logits\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 88
    },
    "id": "89NY9dS37cUi",
    "outputId": "f3a343ef-aeb7-46ef-918b-6695469a89c3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/rnn.py:50: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Le modèle a 120,325,520 paramètres à entraîner\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "#device = 'cpu'\n",
    "def count_parameters(model): \n",
    "    s = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "    print(f'Le modèle a {s:,} paramètres à entraîner')\n",
    "\n",
    "\n",
    "model = Att_BLSTM(word_vec = word_vec, class_num = class_num, max_len = 500, word_dim = 300, hidden_size = 100, layers_num = 1, dropout = 0.5).to(device) \n",
    "\n",
    "count_parameters(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "V2UD8tFm7cVS"
   },
   "source": [
    "### Optimiseur"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "s9hUBK4w7cVZ"
   },
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "criterion = nn.CrossEntropyLoss().to(device)\n",
    "\n",
    "def optimiseur(model, lr=3e-4, eps=1e-6, weight_decay_rate=0.001, second_weight_decay_rate=0.0):\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "            'weight_decay_rate': weight_decay_rate},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "            'weight_decay_rate': second_weight_decay_rate}]\n",
    "    return optim.Adam(\n",
    "        optimizer_grouped_parameters,\n",
    "        lr=lr,\n",
    "        eps=eps\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wgOUQGdF7cV2"
   },
   "source": [
    "### Métriques"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "JUtRFbFa7cV7"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "import math\n",
    "\n",
    "\n",
    "def f1_loss(predict_label, true_label, class_num=class_num):\n",
    "    \n",
    "    assert true_label.shape[0] == predict_label.shape[0]\n",
    "    confusion_matrix = np.zeros(shape=[class_num, class_num], dtype=np.float32)\n",
    "    xDIRx = np.zeros(shape=[class_num], dtype=np.float32)\n",
    "    for i in range(true_label.shape[0]):\n",
    "        true_idx = math.ceil(true_label[i]/2)\n",
    "        predict_idx = math.ceil(predict_label[i]/2)\n",
    "        if true_label[i] == predict_label[i]:\n",
    "            confusion_matrix[predict_idx][true_idx] += 1\n",
    "        else:\n",
    "            if true_idx == predict_idx:\n",
    "                xDIRx[predict_idx] += 1\n",
    "            else:\n",
    "                confusion_matrix[predict_idx][true_idx] += 1\n",
    "\n",
    "    col_sum = np.sum(confusion_matrix, axis=0).reshape(-1)\n",
    "    row_sum = np.sum(confusion_matrix, axis=1).reshape(-1)\n",
    "    f1 = np.zeros(shape=[class_num], dtype=np.float32)\n",
    "\n",
    "    for i in range(0, class_num): \n",
    "        try:\n",
    "            p = float(confusion_matrix[i][i]) / float(col_sum[i] + xDIRx[i])\n",
    "            r = float(confusion_matrix[i][i]) / float(row_sum[i] + xDIRx[i])\n",
    "            f1[i] = (2 * p * r / (p + r))\n",
    "        except:\n",
    "            pass\n",
    "    actual_class = 0\n",
    "    total_f1 = 0.0\n",
    "    for i in range(1, class_num):\n",
    "        if f1[i] > 0.0: \n",
    "            actual_class += 1\n",
    "            total_f1 += f1[i]\n",
    "    try:\n",
    "        macro_f1 = total_f1 / actual_class\n",
    "    except:\n",
    "        macro_f1 = 0.0\n",
    "    return macro_f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0bjvZyh97cWV"
   },
   "source": [
    "## Entraînement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "KCrzVfDv7cWb"
   },
   "outputs": [],
   "source": [
    "def train_model(model, iterator, optimizer, criterion):\n",
    "    predict_label = []\n",
    "    true_label = []\n",
    "    total_loss = 0.0\n",
    "    acc = []\n",
    "    for _, (data, label) in enumerate(iterator):\n",
    "        model.train()\n",
    "        data = data.to(device)\n",
    "        label = label.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        logits = model(data)\n",
    "        loss = criterion(logits, label)\n",
    "        \n",
    "        loss.backward()\n",
    "        nn.utils.clip_grad_value_(model.parameters(), clip_value=5)\n",
    "        optimizer.step()\n",
    "        total_loss += loss.item() \n",
    "\n",
    "        _, pred = torch.max(logits, dim=1) \n",
    "        pred = pred.cpu().detach().numpy().reshape((-1, 1))\n",
    "        label = label.cpu().detach().numpy().reshape((-1, 1))\n",
    "        predict_label.append(pred)\n",
    "        true_label.append(label)\n",
    "        \n",
    "    predict_label = np.concatenate(predict_label, axis=0).reshape(-1).astype(np.int64)\n",
    "    true_label = np.concatenate(true_label, axis=0).reshape(-1).astype(np.int64)\n",
    "    eval_loss = total_loss / predict_label.shape[0]\n",
    "\n",
    "    f1 = f1_loss(predict_label, true_label)\n",
    "    \n",
    "    return f1, eval_loss\n",
    "\n",
    "def evaluate(model, iterator, criterion):\n",
    "       \n",
    "    predict_label = []\n",
    "    true_label = []\n",
    "    total_loss = 0.0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        model.eval()\n",
    "        for _, (data, label) in enumerate(iterator):\n",
    "            data = data.to(device)\n",
    "            label = label.to(device)\n",
    "\n",
    "            logits = model(data)\n",
    "            loss = criterion(logits, label)\n",
    "            \n",
    "            total_loss += loss.item()\n",
    "\n",
    "            _, pred = torch.max(logits, dim=1)  \n",
    "            pred = pred.cpu().detach().numpy().reshape((-1, 1))\n",
    "            label = label.cpu().detach().numpy().reshape((-1, 1))\n",
    "            predict_label.append(pred)\n",
    "            true_label.append(label)\n",
    "    predict_label = np.concatenate(predict_label, axis=0).reshape(-1).astype(np.int64)\n",
    "    true_label = np.concatenate(true_label, axis=0).reshape(-1).astype(np.int64)\n",
    "    eval_loss = total_loss / predict_label.shape[0]\n",
    "\n",
    "    f1 = f1_loss(predict_label, true_label)\n",
    "    \n",
    "    return f1, eval_loss, predict_label, true_label\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "zOJIZA4A7cXF"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs\n",
    "\n",
    "def entrainement(model, optimizer, batch_size = 10, N_EPOCHS = 50):\n",
    "    t_loss = []\n",
    "    t_f1 = []\n",
    "    v_loss = []\n",
    "    v_f1 = []\n",
    "    best_valid_loss = float('inf')\n",
    "\n",
    "    for epoch in range(N_EPOCHS):\n",
    "\n",
    "        start_time = time.time()\n",
    "\n",
    "        train_f1, train_loss = train_model(model, train_loader, optimizer, criterion)\n",
    "        t_loss.append(train_loss)\n",
    "        t_f1.append(train_f1) \n",
    "        \n",
    "        valid_f1, valid_loss,_,_ = evaluate(model, valid_loader, criterion)\n",
    "        v_loss.append(valid_loss)\n",
    "        v_f1.append(valid_f1)\n",
    "        end_time = time.time()\n",
    "\n",
    "        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "\n",
    "        if valid_loss < best_valid_loss:\n",
    "            best_valid_loss = valid_loss\n",
    "            torch.save(model.state_dict(), 'tut2-model_ner.pt')\n",
    "\n",
    "        if epoch%10 == 0: \n",
    "            print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "            print(f'\\tTrain Loss: {train_loss:.3f} | Train F1 score: {train_f1*100:.2f}%')\n",
    "            print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. F1 score: {valid_f1*100:.2f}%')\n",
    "    return t_loss, v_loss, t_f1, v_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "id": "z2WTKauA7cXh",
    "outputId": "3964ba8d-0cee-43b3-b353-d47483cc5535"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 0m 28s\n",
      "\tTrain Loss: 0.280 | Train F1 score: 7.11%\n",
      "\t Val. Loss: 0.256 |  Val. F1 score: 44.89%\n",
      "Epoch: 11 | Epoch Time: 0m 29s\n",
      "\tTrain Loss: 0.141 | Train F1 score: 57.32%\n",
      "\t Val. Loss: 0.107 |  Val. F1 score: 68.60%\n",
      "Epoch: 21 | Epoch Time: 0m 28s\n",
      "\tTrain Loss: 0.106 | Train F1 score: 69.09%\n",
      "\t Val. Loss: 0.091 |  Val. F1 score: 74.19%\n",
      "Epoch: 31 | Epoch Time: 0m 28s\n",
      "\tTrain Loss: 0.083 | Train F1 score: 75.97%\n",
      "\t Val. Loss: 0.089 |  Val. F1 score: 76.09%\n",
      "Epoch: 41 | Epoch Time: 0m 28s\n",
      "\tTrain Loss: 0.063 | Train F1 score: 81.50%\n",
      "\t Val. Loss: 0.091 |  Val. F1 score: 77.52%\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 50\n",
    "\n",
    "t_l, v_l, t_f1, v_f1 = entrainement(model, optimiseur(model, lr = 1e-4, weight_decay_rate = 0.001), N_EPOCHS = N_EPOCHS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 509
    },
    "id": "0wR0Cj8_7cX4",
    "outputId": "f0620869-08f1-4b4d-98f7-9b595f133037"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAAHsCAYAAAAQKp5gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxU9dnw/885Z7Ysk32STFYS9jXsqIhFRUFFQr1FlNYutlgftLTa9pFuoKL9Fe+79rZWvauttlbtbdU+ooiKawVkk13CmoQEyGTf18nMOb8/JgxEEBJmkkzI9X698uLMzJkz37mY5JrvrhiGYSCEEEKIkKb2dQGEEEIIcX6SsIUQQoh+QBK2EEII0Q9IwhZCCCH6AUnYQgghRD8gCVsIIYToByRhCyGEEP2AJGwhBoirrrqKzz77rK+LIYS4QJKwhRBCiH5AErYQA5jb7eaRRx7h8ssv5/LLL+eRRx7B7XYDUF1dzQ9+8AMmT57M1KlTWbRoEbquA/DMM88wY8YMJkyYwOzZs9m0aVNfvg0hBgRTXxdACNF3nn76aXbv3s3q1atRFIUlS5bw1FNP8eMf/5jnn3+epKQkfzLevXs3iqJQUFDASy+9xGuvvUZSUhLHjx/3J3IhRM+RGrYQA9hbb73F3XffTXx8PHFxcdx99928+eabAJhMJioqKigpKcFsNjN58mQURUHTNNxuN/n5+bS3t5OWlkZGRkYfvxMhLn6SsIUYwMrLy0lJSfHfTklJoby8HIDvfe97ZGZmcscdd3D11VfzzDPPAJCZmckvfvELnnjiCS677DLuvfdeysrK+qT8QgwkkrCFGMASExMpKSnx33a5XCQmJgIQGRnJsmXL+PDDD/mf//kfnn/+eX/z+I033sg//vEPPv74YxRF4b/+67/6pPxCDCSSsIUYQNrb22lra/P/3HDDDTz99NNUV1dTXV3Nk08+yY033gjAxx9/TFFREYZhEBkZiaZpqKpKQUEBmzZtwu12Y7FYsFqtaJrWx+9MiIufDDoTYgC58847O92+4447GDNmDPPmzQNgzpw5LFmyBICioiJWrlxJdXU1UVFR3HbbbUybNo0DBw7wu9/9jvz8fMxmMxMmTOChhx7q9fcixECjGIZh9HUhhBBCCHFu0iQuhBBC9AOSsIUQQoh+QBK2EEII0Q9IwhZCCCH6AUnYQgghRD8gCVsIIYToB/rNPOyamiZ0PTgz0OLjI6mqagzKtQYqiWHgJIaBkxgGRuIXuGDGUFUVYmMjvvLxfpOwdd0IWsI+eT0RGIlh4CSGgZMYBkbiF7jeiqE0iQshhBD9gCRsIYQQoh/oN03iQggheobX66GmpgKPx93XRel3ystVdF3v9vNMJguxsQ40retpWBK2EEIMcDU1Fdhs4UREJKMoSl8Xp18xmVQ8nu4lbMMwaGqqp6amgoQEZ5efNyCbxL0yyEIIIfw8HjcREVGSrHuJoihERER1u0VjwCXs7QfLWfyb92lube/rogghRMiQZN27LiTeAy5hJ0SHUVHTwqe7XX1dFCGEEKLLBlwfdmayndHZ8Xy4/RjXTElDUwfcdxYhhAhZixd/m/b2djyedo4dKyYrazAAw4YN5xe/WNGla7zxxmu0tbWxcOE3uvXa99xzJ7fddjvTp8/odrl7w4BL2AC5VwzmN3/dyo5DlUwZkdjXxRFCCNHh2Wf/BoDLVcL3v387f/3ry2ec4/F4MJm+On3Nn39zj5WvLw3IhD11dDKOGBvrthVLwhZCiH7g5ptvZO7cXLZv30ZKSip33rmEBx74JU1NTbjdbi67bDpLlvwIgL/85U+0tLRwzz0/Zu3at3j//Xex26MoKMjHbo/k4YcfJT4+ocuvvXnzZ/zpT39E13ViYmL52c9+QVpaOsXFR/nNbx6kpaUVXfdy3XU3smjR7axf/wnPPvs0qqrh9Xq4997/y8SJkwOOwYBM2JqqMGtyOv/44DD5JXUMTonu6yIJIURI2LjXxYY9PTPG5/JxTqaP7fo0pi+rrKzkiSf+BEBbWxurVv2e8PBwPB4P9913D5s3f8Yll1x2xvP278/jb3/7B0lJyaxa9TCvvfYKP/jB3V16zZqaah5+eDlPPPEMWVnZrFnzBg8++CueffZv/Otfr3HZZZfzrW99D4D6+noA/vznP/GTnywjJ2cCXq+X1taWC37PpxuwHbiXj3USZtV4f9uxvi6KEEKILpgz5wb/sa7rPPXU43z727fxve99k4KCfA4fPnTW540bl0NSUjIAo0ePoaTkeJdfc9++Lxg8eBhZWdkAXH/9PI4cOURzcxPjx0/grbdW8+yzT7N9+zbsdjsAkyZN5o9//D0vv/wCRUWFREREXuhb7mRA1rABwqwmrshJ4f1tx6ma2Up8tK2viySEEH1u+tjAasE9KTw8zH/8yisv0dBQzzPP/BWr1cqqVY/gdred9XkWi8V/7Gum9nbjVQ2+agbWzJlXk5Mznk2bPuPFF//K22+/yfLlK1m69Cfk5x9h+/Zt/PrXy1i48BvMm/f1brzm2Q3YGjbA1ZPSMDD4cEfXv20JIYToew0NDcTHJ2C1WqmoKGfDhn/3yOuMHj2OI0cOUVR0FIB33lnD0KHDCQ+P4PjxY8THx3P99Tfy3e8uJi9vHwDFxUcZPHgIt9xyG9deex379+cFpSwDtoYNvjnZk4Yn8u9dJcybPgibZUCHQwgh+o0FC27l17++n+9+dxGJiUlMmjQlKNf9zW8ewGKx+m//538+zq9+9RAPPvhLvF4vMTGxLF++EoCPPnqf999/F5PJhKIo/OhHPwHg6af/yPHjxWiaicjISH7+8+VBKZtiGEa/WKezqqoxaHuOOhx2KioaADhyoo7f/H0737hmGFdPSgvK9QeC02MoLozEMHASw8CcjF9paRHJyZl9XZx+6ULWEj/py3FXVYX4+K/u7x7QTeIAQ1KjyU6J4v3Pj6H3j+8uQgghBqABn7ABrp2STnlNC7uPVPZ1UYQQQoizkoQNTBruIC7KKlO8hBBChCxJ2ICmqlw9KY0DxbUUl0l/mBBCiNAjCbvD13JSsJo11kktWwghRAiShN0h3Gbm8rFOtuSVUdt49sn3QgghRF+RhH2aWVPS0HWDj3ec6OuiCCGEEJ0MuISt17qo3fQGZ5t+nhQbTs6QBD7eeQJ3e3eWrhNCCBEM9933Q9544/VO9xmGwYIF89i1a8dXPu+RRx7g9ddfAXz7Yb/yyktnPW/t2rf41a/+71kfu+eeO9m4cf0FlrznDbiE7Sk9RPVHf8doqDjr49dMSaexpZ1tB8p7uWRCCCFuuGEea9e+1em+nTu3o2ka48dP7NI15s+/mYULv9ETxetTXVqLs7CwkGXLllFbW0tMTAyrVq1i0KBBnc558sknWbt2LZqmYTKZuPfee5kxYwYATzzxBC+//DKJib69pydOnMiKFSuC+066SEseCoCnZD+WqDP3wh6REUNibBgb9rhCdgF8IYToKe2HNtJ+8NMeubZ5+BWYh00/5zlXXDGTxx77LYWFBf4dst5++02uv/5G8vOP8Lvf/ZbW1hbcbjfz5n2dW25ZdMY1Tt8Pu729nd///lF27tyOw5FIRsagbpe7X+2HvWLFChYtWkRubi6rV69m+fLlvPDCC53OGTduHHfccQdhYWEcOHCAb37zm2zYsAGbzbcL1vz587n//vsDLnCg1GgnWkQM3pIDMOJrZzyuKAozxjl5/d8FlNU0kxQb3gelFEKIgclsNnPNNXN45523WLLkRzQ3N7F+/b+56657iIiI4L//+yksFgvNzc3ceee3mTr1UgYNyvrK661e/TouVwl///s/8Xg83H33YpzOrlfGQmk/7PMm7KqqKvLy8nj++ecBmDt3LitXrqS6upq4uDj/eSdr0wDDhw/HMAxqa2tJTk4OSkGDRVEUbJmjaS7KwzAMlLPsm3bZGCf/+rSAjXtd3HTF4D4opRBC9A3zsOnnrQX3tBtuyOWnP/0hd955Nx9++D7jxuXgcCRSXV3FH//4W44cOYSiqFRWVnDkyKFzJuwdO7Zz3XVzMZlMmEwmZs++jj17dnW5LGfbD/t3v1vl3w/7yScfp63NzcSJk/216JP7YV955SwuueQysrOHBBaQDuftw3a5XCQlJaFpGgCappGYmIjL5frK57zxxhtkZGR0StZvv/02N954I3fccQc7d+4MQtEvXFjmGIymGoz6s/dTx9qtjM2OZ+Pe0qBtOCKEEKJrhg4dRnx8Alu2bGLt2je54YZ5APzpT08SFxfPc8+9xN/+9g9GjhyN2+0+57UC39/q3PthP/PM86SmpvHii39l5UrfrlxLl/6EZcuWYzKZ+fWvl/Hmm/8vwDL4BH0/ya1bt/L444/z3HPP+e+79dZbueuuuzCbzWzcuJElS5awdu1aYmNju3zdc+1g0l1udTQAYQ1HiRpy9m8+11+ezW//to3jNS1MGpEUtNe+mDgc9r4uQr8nMQycxDAwDoed8nIVkym0xiDfeGMuzz//DKWlLmbOvBKTSaWpqZFhw4Zhs1nIzz/Cnj27mD37OkwmFUVRUFUFk0lFVU8dT506lXXr1nLttbPxeDx88MF7JCUln/X9KoqCpimdHsvJGc9vf7uS48eLGDQoizVr3mLYsOFERdk5dqyY1NQ05s3LJTMzk4cffgCTSaWo6CjDhw9j+PBhtLW1cPBgHibTf5zxeqqqduvze96E7XQ6KSsrw+v1omkaXq+X8vLys/YB7Ny5k5/97Gc89dRTZGdn++93OBz+4+nTp+N0Ojl8+DBTp07tckGDub1mQkIKSlg0tYd20pY27aznZCdGEBlmZs36AjLipR/7y2Rbw8BJDAMnMQzMyfjpun7BW0T2lKuvnsMTTzxObu5NKIqGx6PzrW/dwcqVy3nnnbWkpqaSkzMBXTfweHQMw/Af6/qp47lzv86hQ4e57babSUxMIidnIi7XibO+X8MwWLlyxVn3w16+/Bf+/bB//euVeDw677+/rtN+2EuX/gSPR+ePf/zDGfthn+31dF3v9Pk93/aaXdoP+/bbb+fmm2/2Dzp77bXX+Pvf/97pnD179rB06VIef/xxcnJyOj1WVlZGUpKvlrp//36+853vsGbNmk6J/HyCvR928f8+itd1kIhv/P6s/dgA//jgMB/tOM5j90zHHm4JymtfLOQPZeAkhoGTGAZG9sMOXG/uh92lJvEHHniAZcuW8dRTTxEVFcWqVasAWLx4MUuXLmXs2LE8+OCDtLa2snz5cv/zHn30UYYPH85jjz3Gvn37UFUVs9nMo48+2q1k3RM05wg8+Vsw6stQos8+MG7GOCfvf36MzXllXDM5vZdLKIQQQpzSpYQ9ePBgXn311TPuf/bZZ/3Hr7/++hmPn3QywYcSU8pI2gBPyQEsX5Gw0xIjyXLaWb/bxaxJaV9ZExdCCCF6WmiNMuhFSnQSSngMXteBc553+bgUjlc0UiTbbgohLmKBj6YW3XEh8R64CVtR0Jwj8JYcOGfgpo1MxGxSWb/nq6exCSFEf3ZyRS7Re7xeD6qqdes5AzZhA2gpIzCaazHqyr7ynHCbmUnDHWzZVyYbggghLkphYZE0NNRiGKE1UvxiZRg6DQ01hIV1b7py0Odh9ycm5whfP7brAJaYr16RbcZYJ5v3lbHjcAWXjAqtlduEECJQkZHR1NRUUFZ2HJCm8e5QVRVd7+4XHQWLxUZkZHS3njWgE7a/H7vkAIyc+ZXnDc+MJSHaxoY9LknYQoiLjqIoxMWduRmSOL/enFo4oJvEFUVBSxmBt2T/OfuxVUXh8rFO9h+tobI2OIu4CyGEEN0xoBM2+OZjGy11GHWl5zzv5FabG78493lCCCFETxjwCduUMgLwzcc+l/hoG6Oy4tiwx4Uu0x+EEEL0sgGfsJWors3HBt/KZ1X1rRwoqumFkgkhhBCnSMJWFLSUkeedjw0wYWgCETaTzMkWQgjR6wZ8woaO+dgtdeh1507EZpPGJaOS2X6wgqbW9l4qnRBCCCEJG/DNxwZ807vO4/JxTjxenU0y+EwIIUQvkoQNKFGJKBGxXUrYmcl2BqdE8f7nx/B2e7K8EEIIcWEkYXPauuKu8/djA1x/SSYVta18fqCiF0onhBBCSML28/Vj16PXnn9AWc7QBJzx4azdXCQ73AghhOgVkrA7mFJGAnRpepeqKFw3LZNj5Y3sK6zu6aIJIYQQkrBPUuwOlIi4LvVjA1wyOolYu5W1m4t6uGRCCCGEJGw/Xz/28C73Y5s0ldlT0jlQXEv+ibpeKKEQQoiBTBL2abrTjw1wxfgUImwmqWULIYTocZKwT+Pvxy7Z36XzbRYTV09KY+fhSkoqm3qyaEIIIQY4Sdin8fdjd2Hg2UlXT0rDYlJ5d0txD5ZMCCHEQCcJ+zSn9sfuWj82gD3cwoycFDbtK6W6vrWHSyiEEGKgkoT9JSbnCIzWBvSaE11+zuyp6RgGrNt2rAdLJoQQYiCThP0lWtoYADxHd3T5OQnRYUwblcS/d5XQ2CKbggghhAg+SdhfokbGoSYNwVO4rVvPu+6SDNravXy043gPlUwIIcRAJgn7LMxZU9CrjqHXdX1HrjRHJDmD4/ng8+O0tXt7sHRCCCEGIknYZ2HKngxAe0H3atnXX5pJY0s7G/Z0bR63EEII0VWSsM9CjYxHTRyMp+Dzbj1vaFoMQ9KieXdLMR6vbL0phBAieCRhfwVz9hT0qiL0+vJuPe/6SzKpqm9l2/7uPU8IIYQ4F0nYX8GUPQWA9oKt3XreuMHxpDoiWL2hELf0ZQshhAiSLiXswsJCFi5cyOzZs1m4cCFHjx4945wnn3ySG264gXnz5nHTTTexfv16/2Ner5cHH3yQWbNmcc011/Dqq68G7Q30FF+zeDaebvZjq4rColnDKK9tYfXGwh4qnRBCiIGmSwl7xYoVLFq0iPfee49FixaxfPnyM84ZN24cr732Gm+++Sa/+c1vuPfee2lt9a389dZbb1FcXMy6det45ZVXeOKJJzh+PPSnP5mzpqBXdr9ZfGRmLFfkOHlvyzGKSht6qHRCCCEGkvMm7KqqKvLy8pg7dy4Ac+fOJS8vj+rq6k7nzZgxg7CwMACGDx+OYRjU1tYCsHbtWhYsWICqqsTFxTFr1izefffdYL+XoLvQ0eIAC64cgj3czF/fOYBXlwFoQgghAmM63wkul4ukpCQ0TQNA0zQSExNxuVzExcWd9TlvvPEGGRkZJCcn+6+RkpLif9zpdFJa2vU5zgDx8ZHdOv98HA57F06yc8I5BKN4B45rbu3e9YH/c3MOv/3bNj7Lq+CmK4dcWEFDWJdiKM5JYhg4iWFgJH6B660Ynjdhd9fWrVt5/PHHee6554J63aqqRnS9axtynI/DYaeiootN1RkTcW/5J2X5+ahRid16naHJkUwYmsBL7+5neKqdxNjwCyhtaOpWDMVZSQwDJzEMjMQvcMGMoaoq56ycnrdJ3Ol0UlZWhtfrG/Hs9XopLy/H6XSece7OnTv52c9+xpNPPkl2dnana5SUlPhvu1wuf+071JmyTo4W796cbPDt/vXNa4ejaQp/e/dgl3cAE0IIIb7svAk7Pj6ekSNHsmbNGgDWrFnDyJEjz2gO37NnD/feey9/+MMfGD16dKfH5syZw6uvvoqu61RXV/PBBx8we/bsIL6NnqNGOVAdWd1eW/ykWLuVBTOHsL+oho17u9cNIIQQQpzUpVHiDzzwAC+++CKzZ8/mxRdf5MEHHwRg8eLF7N27F4AHH3yQ1tZWli9fTm5uLrm5uRw8eBCA3Nxc0tLSuPbaa7nlllu4++67SU9P76G3FHymrCnoFYXo9RUX9PwrxqcwLC2aVz46TF2TO8ilE0IIMRAoRj9pp+2zPmxAr6+g6X9/hnXaLVhyrr+g13RVNbHiua1MHObgrtwxF3SNUCJ9X4GTGAZOYhgYiV/gQqoPW3Q0iycMuqDpXSc54yO4cXoWW/eXs+twZRBLJ4QQYiCQhN1FpuyOZvGGC2sWB7huWgZpjgj+vu4gLW2eIJZOCCHExU4SdheZO9YW7+4OXqczaSrfuW4ktQ1tvPbv/GAVTQghxAAgCbuL1KhE1PjMgJrFAbJTorhmSjof7zjBpi9k1LgQQoiukYTdDb5m8QL0hsD6oG+eOZiRmbE8t3Y/B4trglQ6IYQQFzNJ2N3gbxYvvPBmcfA1jd/99TEkxobxxOt7cVU1BaN4QgghLmKSsLtBjU5Cjc8IuFkcINxm5t4FOZg0hd//czf1Mj9bCCHEOUjC7iZT9hT08nz0xqqAr5UQE8aPFuRQ3+TmD6/vwd3uDUIJhRBCXIwkYXfTyWbx9iNbgnK9LGcUd84bTWFJPc+uyUPvH+vYCCGE6GWSsLtJjU5GSxlJ+973MDzBacaeOMzBwquGsP1gBa99LNO9hBBCnEkS9gWwTJyH0VJH+4FPg3bNa6akc/XENN7dWszHO44H7bpCCCEuDpKwL4DmHIGWPAz37rcxvO1BuaaiKNw2ayg5g+N58f1D7MmX5UuFEEKcIgn7AiiKgmViLkZTDe0H1wftuqqq8IPc0aQnRvL0G/v4ojDwgW1CCCEuDpKwL5CWOgo1aQjuXW9jeIO3LrjNYuLHC3JIiLHx+3/u5r2txfSTDdWEEEL0IEnYF0hRFKwT52E0VtF+eGNQrx0TaeWXt09i4lAHr3x0hD+v2S9TvoQQYoCThB0ALW0sqiML9841GHpwd9+yWUz8n6+PYf6MLDbtK+W3L+2gur41qK8hhBCi/5CEHQBfLTsXo6ECz+FNQb++qijMm57FD28ai6u6mYf+9jlHjtcF/XWEEEKEPknYAdIyclDjM2nbuQZD75lm6wnDHPzq9knYLBqrXt7Bp7tLeuR1hBBChC5J2AHyjRifh1Ffhic/OKufnU2qI5Jff3syIzJj+es7B3hx3UE8Xr3HXk8IIURokYQdBKZBE1Dj0nDveBND77kkGmEz8+MF45gzNYOPdpzg/3txO6XVzT32ekIIIUKHJOwgUBQVy8R56HWleAq29uhraarKLVcNYcn8MZTXtPDAc1v5eOcJmfolhBAXOUnYQWLKmowam4J755sYRs83VU8ekchD35vG0PQY/v7eQR5/bQ91jW09/rpCCCH6hiTsIFEUFcuEeeg1JXgKt/fKa8bardx7Sw6LZg1lf1ENv/7LVnYequiV1xZCCNG7JGEHkSl7Kkp0sq8vuxdq2eCb+jVrcjrLvzOFOLuVJ/61l+fX7qelLbjzwoUQQvQtSdhBpKgq1gk3olcfw1O0s1dfOzUhgl99ezLXX5LJhj0uHnh+KweLa3q1DEIIIXqOJOwgMw25BCUqibYtr2K0926fsklTuXnmYO7/xkQMA1a9vJM/vLaHE5VNvVoOIYQQwScJO8gUVcM249sYdWW0ffZSn5RhWHoMK78/jf/4WjYHj9Ww/C9b+Os7+6lpkEFpQgjRX5n6ugAXI1PqKCzjr8e962209DGYs6f2ehmsZo0bLh3EFTkprPmsiI92HGfzvjKumZLOddMyCbfJf70QQvQnUsPuIZbJX0d1ZNP66V/RG/tuX2t7uIXbZg3lkTsvYeIwB29vKmLZnzaxbtsx2j2yUpoQQvQXkrB7iKKaCLv6LjB0Wj/6U4+tM95ViTFh3DlvNCu+M4WMpEj+98PDLP/LFopKG/q0XEIIIbqmSwm7sLCQhQsXMnv2bBYuXMjRo0fPOGfDhg3cdNNNjBkzhlWrVnV67IknnuDSSy8lNzeX3NxcHnzwwaAUPtSpUYnYLv8W3tJDuHe+1dfFASAz2c5Pb53Afbfk4PboPPL3z/lox3FZKU0IIUJclzoyV6xYwaJFi8jNzWX16tUsX76cF154odM56enpPPzww7z33nu43e4zrjF//nzuv//+4JS6HzEPvQzP8S9w71iNljoKU/Kwvi4SAGOy43ngu1P485r9vLjuEAeLa/nOdSMIs0rfthBChKLz1rCrqqrIy8tj7ty5AMydO5e8vDyqq6s7nZeZmcmoUaMwmeQP/pfZpt+OYnf4msbbQmeKlT3cwo8WjOPmmYPZfrCCB5/fJk3kQggRos6bXV0uF0lJSWiaBoCmaSQmJuJyuYiLi+vyC7399tts2LABh8PBD3/4QyZMmNCtgsbHR3br/PNxOOxBvd652Wn9j59Q8rdfYGx9EcfXf4KiKL34+uf27RvHMHm0k/988XMe+ft2Fs8fw3WXDjpvGXs3hhcniWHgJIaBkfgFrrdi2CvV4VtvvZW77roLs9nMxo0bWbJkCWvXriU2NrbL16iqakTXg9PP6nDYqajo5ZqkOQnL5Jto2voqJY63sYz4Wu++/nkk2i0s//Zk/rxmP0+/vofP95Wes4m8T2J4kZEYBk5iGBiJX+CCGUNVVc5ZOT1vk7jT6aSsrAyv1zfK2ev1Ul5ejtPp7HIhHA4HZrMZgOnTp+N0Ojl8+HCXn3+xsORch5Y6irbPXsJbW9LXxTnD2ZrIN+0rxeOV6V9CCNHXzpuw4+PjGTlyJGvWrAFgzZo1jBw5slvN4WVlZf7j/fv3c+LECbKysi6guP2boqjYZi5G0Sy0fvg0hrulr4t0BlVRuP6STP7vogmYTCrPvpXHz/+0iXVbi2VDESGE6EOK0YX5PPn5+Sxbtoz6+nqioqJYtWoV2dnZLF68mKVLlzJ27Fg+//xz7rvvPhobGzEMA7vdziOPPMKMGTO4//772bdvH6qqYjabWbp0KV/7WveahPt9k/hpPMf20PLuf6M5hxM2514Uk6XPynIuumGwN7+Kd7YUc+hYLeFWE1dOTGXWpDSGZCVIU1qA+vpzeDGQGAZG4he43mwS71LCDgUXU8IGaD/8Ga0fP4MpcwK2a+5BUbU+Lc/5FJTU8+6WIrYfqkBTFa6anMHXxiXjjI/o66L1W6HwOezvJIaBkfgFrjcTtszB6iPmoZdhuJtp2/girf/+C7aZ30dRQnfhueyUKJZ8fSxlNc2s23qMT7YfY92WIkZmxnLZmGQmDXdgs8jHSQgheor2wAMPPNDXheiKlhY3wWoLiIiw0tx85uIuvU1LzAZFpf2LdRhtTWjpY0NqutfZRIaZyRmSwN3WJ+wAACAASURBVE1XD8PwejlYXMuGvS7e//wYrqpmbFaNhChbyL+PUBAqn8P+TGIYGIlf4IIZQ0VRCA//6i5SqRL1McuEGzHammjf+x6KNQLr5K/3dZG6JDrSyo3Ts5h72SDyT9Tz2Rcutu4vZ9O+UmLtVi4ZncRlY5ykJkiTuRBCBIMk7D6mKArWS24FdzPuHatRrOFYxs7u62J1maIoDEmLZkhaNLfNGsquI1V8ttfFe1uO8c7mYrKcdmaMS2HaqCRZ9lQIIQIgf0FDgKIoWGd8F8PdQtumf6BYwjEPn9HXxeo2s0ljyohEpoxIpL7Jzea8MjbsKeGF9w7yvx8eZsqIRGbkpDA0LVqazIUQopskYYcIRVWxXfUDWt5rpfXT58ASjjlrUl8X64JFRVi4dko610xO42hpA5/uLmFLXhkbvyglKS6cGeOcTB+TTHSkta+LKoQQ/YJM6woxRnsbzWv/E73iKNZpt2AeMyskR49fSAzb3F62HShn/Z4SDh+vQ1UUJg538B9XZJMUF95DJQ1dofw57C8khoGR+AVOpnUNYIrZSvice2n56E+0bXoZz9Ht2L72PdSoxL4uWsCsFo3Lxzm5fJwTV1UT63e7+HjXCXYequDKCanMuzyLyDBzXxdTCCFCkkzrCkGKyYJpyCWo9gTaD26gPe9DFEsYquP8O2j1lkBjaA+3MDorjhljnTS3efhk1wn+vbMEVVXITLajqaHxPntSqH8O+wOJYWAkfoHrzWldodfWKgDff5x5+AwiFjyMljyMto0v0vL2f6I3VPR10YIqOtLKt+eM4ME7pjI4NZp/fnyEXz67ma37y+gnvTVCCNErJGGHODUynrDrfoL1iu/irSik6bVf4877+KJLZmmOSO69JYefLByPzaLxP6v38ZsXt3PkeF1fF00IIUKCDDrrR/SGSlo/fQ7viTy01NHYrvgOqt3RJ2XpyRjqusGGvS7+36cF1DW5GZwSxcwJqUwZkYjFHNprrndHf/0chhKJYWAkfoGTzT/OQhK2j2EYtO//mLYt/wTdi2X8XCw51/X6jl+9EcNWt4dPd7v4964TuKqaibCZmD7WydfGp1wUm470589hqJAYBkbiFzhJ2GchCbszvbGats3/i6dgK4rdge2yRZgyJ/Ta6/dmDA3D4GBxLZ/sOsH2gxV4dYMRGTHMnJDKxGEOTFr/7Nm5GD6HfU1iGBiJX+BkWpc4LzUyjrBZS/CcmEnbZy/S8t7jaBk52C5dhBqd1NfFCypFURiRGcuIzFjqmtxs2FPCJztL+J/V+4gKNzNqUBxZKVFkO6PISLJjNvXPBC6EEOciNeyLgKF7aP/iA9q2vwFeD5ac67BMmIti6rlVxPo6hrpu8EVhNRv2usg/UUdNQxsAmqqQkRRJtjOarBQ72SnRJMWGhcx0uNP1dQwvBhLDwEj8Aic1bNEtimrCMm4OpiGX0Lb5Fdw736L98GdYL1mIKWtKSCarQKmqwrjB8YwbHA9ATUMbBSV1FJTUU+iqZ8NeFx/uOA74FmxJS4gg1RFJmuPUv/ZzzHcUQohQIzXsi5DHdZC2jS+iVx9DTRzsS9zJw4L6GqEeQ103KKlsosBVz7GyRo5X+H6aWj3+c6IjLKQ6IhiSGs3l45wkRIf1ahlDPYb9gcQwMBK/wMmgs7OQhN09hq7jObSBts//hdFci2nQRKxTF6DGOINy/f4YQ8MwqGty+5J3eRMnKn3/Fpf53sfYwfFcOSGVsdnxqL2w0lp/jGGokRgGRuIXOGkSFwFTVBXziCswDZ6Ge+97uHevxfPqLzGPnIllYi5qeHRfF7HXKYpCTKSVmEgrY7Li/fdX1rXw6W4X63eX8Phre4iPsnLF+FSuGOeU3cSEECFDatgDhN5Sj3v7G7Tv/wRMFiw512MZOxvFfGEJ6WKMocers+twJR/vPMH+oho0VWHCMAdfG5/CiIwYNDW4o88vxhj2NolhYCR+gZMatgg6NSwK2+XfwjLmGtq2vob783/h3vsepsyJmLMnoaWORtEG9k5ZJk1l8ohEJo9IpLS6mX/vOsGGPS4+P1BOZJiZ8UMSmDjMwahBsRfVimtCiP5BatgDlKf0MO15H+Ep2gXtLWAOw5SZgylrMqb0seedEjZQYtju8bL7SBU7Dlew+0gVLW0erGaNMdlxTBzmIGdwPOG2C/uiM1Bi2JMkhoGR+AVOatiix5mSh2JKHorhbcd7Yh+ewu20H92B58hmMFkwpY/zJe+MHBRL746eDiVmk+avdXu8OgeKa9h5qJIdhyvYfrACTVUYnhFDZpKdxNgwkmLDSYwNI8ZuRb0Ip9MJIfqO1LCFn6F78boO4inYhufodoyWelBNaGmjMQ+ahDZoAqrNDkgMdcOgsKSeHYcq2FNQRVl1Mx7vqc+nxaTiiA0jMSaMpLhwhqXFMCY7rtMyqgM9hsEgMQyMxC9wMq3rLCRh9y5D1/GWH8FTuB1P4ecYjVWgKGjOEZgGTSJp0hXUtMnCIyfpukF1fStltS2UVzdTVtNCeU0LZTXNVNS24PEaRNhMTBruYNrIJIZnxJKUFCWfwwDJ73JgJH6Bk4R9FpKw+45hGOhVRR3Jezt6bQkAamI25qGXYR56GYolvI9LGbo8Xp28o9VsyStjx+FK2txeoiMsXDExjXGDYslOibooV6PrDfK7HBiJX+AkYZ+FJOzQ4a0pwVrxBXV716NXHQPNgmnwNCwjv4aaOFiSzzm0tXvZm1/Flrwy9hRU0e7RSYi2MX5IApnJdjKT7DgTwoM+hexiJb/LgZH4BU4S9llIwg4tJ2PorThK+/5PaM/fDO2tqHFpmEfMxDz0UhRr/9+zuieFR9p4f1MhW/aXcai4FrdHB3zTy9IcEWQm28lIspORFEm6I1Kmkp2F/C4HRuIXuJBL2IWFhSxbtoza2lpiYmJYtWoVgwYN6nTOhg0beOyxxzh06BC33347999/v/8xr9fLww8/zPr161EUhTvvvJMFCxZ0641Iwg4tX46h4W6hPX8L7fs/Qa882lHrnoI5awpqYjZqWFTfFTZEnR5DXTcorW6mqKyB4rIGissaKSptoLnNt/a5pipkpUQxIiOWkZmxDEmNwmySBC6/y4GR+AUu5KZ1rVixgkWLFpGbm8vq1atZvnw5L7zwQqdz0tPTefjhh3nvvfdwu92dHnvrrbcoLi5m3bp11NbWMn/+fC699FLS0tIu4C2JUKRYwrCMnIll5ExfrfvAJ7Qf2Yzn0Ebf4/YENEc2WmIWqiMbLWHQBa+ydjFSVYWUhAhSEiK4dHQy4Bs7UFXXSlFZIwWuOg4U1fL2pqOs+ewoJk1lSGqUb5/wDF8/+Okj0IUQF5/zJuyqqiry8vJ4/vnnAZg7dy4rV66kurqauLg4/3mZmZkAfPjhh2ck7LVr17JgwQJUVSUuLo5Zs2bx7rvv8v3vfz+Y70WECM0xCM3xHayX3oa34ih6RQHe8kK8FQV4Crb6TlIU1Ng0tMRstKQhaMnDUKISpf/7NIqikBATRkJMGJOGOwBobvVw6HgtB4pqOFBcw+r1hbxBIRazyuhBcYwfmkDO4ASiImQEvxAXm/MmbJfLRVJSEprma37TNI3ExERcLlenhH2+a6SkpPhvO51OSktLL7DIor9QTFZMzuHgHO6/T2+pP5XAy/NpL9hG+4F/+84Pi/Il76ShaMlDURMyB/xyqV8WbjMxfkgC44ckANDY0s7B4lryiqrZfaSSnYcrUYDBadFMGJLA+KEJOONlLIEQF4N+s9LZudr1L4TDYQ/q9QaiC4uhHTJSgRkAGIZOe+VxWo8doPX4QVqPH6Dt6A4AFM2MJTkbc2wSpuhETNEOTNEOzDEOtKgEVFP/r0UG+jl0AFkZccy5PBvDMCg4UcfWfaVs3lfKq5/k8+on+aQ6Ipg22sm0McmMyIzrla1De5P8LgdG4he43orheRO20+mkrKwMr9eLpml4vV7Ky8txOru+r7LT6aSkpIRx48YBZ9a4u0IGnYWW4MYwFtIvRUm/lDBAb67FW3YEb+lhPJVHcR/dh9G0Hr40PlIJi0axJ6DGJKPFpqJ2/CiRcShK6Pfn9sTnMMqqMWtiKrMmplJd38rOw5XsOlLJ6k/z+dcnR4iOsDBhqG8TkxGZsf2+31t+lwMzUONnGAZGQwXeyiLUqETUuHSULk6lNAwdveIonqM78BTvIiJzFExZFJRyBTzoLD4+npEjR7JmzRpyc3NZs2YNI0eO7HJzOMCcOXN49dVXufbaa6mtreWDDz7gpZde6vLzxcCihsegZk3GnDXZf5+hezCaatAbqjAaq9AbKzEaKtEbKvEe3+cf3AaA2YYa40SNTfUl8uhkX2K3JwyoddHjomxcPSmNqyel0dzqYW9BFdsPVbBpXxmf7CohzGoiZ3A8E4c5GJMdh83SbxrchOgWQ9fRa47jdR3CW+r7MZprT51gDkNLGoyWPAwteSiaI7vToFjD48Z7Ig9P0U48RbswWupAUdGShxExbApNvfQ+ujStKz8/n2XLllFfX09UVBSrVq0iOzubxYsXs3TpUsaOHcvnn3/OfffdR2NjI4ZhYLfbeeSRR5gxYwZer5eHHnqIjRt9f1QXL17MwoULu1VQqWGHllCLodHaiLe2BL36BHrNCfSOY6OlrvOJ1ghUewKq3XEqiYfHophtvl9Qs63j2AZma4/2ofdVDNs9XvYdrWHHoQp2Ha6ksaXdP/fbbFIxaSd/FP+x2aQQZjWR5YxiSGo0cVG2Xi/32YTa57C/CcX4GYYB7mb0xmowdBRLmG8lRYsNRf3qL5WGtx2jtRGjpR6jtQGjtdH3hb70EN6yw+BuAUCJiPMlZucwtIRB6HWleEsP4y07jF59AjBA0VATMtGSBmM0VOI5vg+8bjDbfBsjZY7HlD4OxRYZevOwQ4Ek7NDSX2JotDai15ejN1ai11diNPpq5UZDBXpDJXjbz30BVUOxhKNExKBExKFGxKFEfunfiFiUC+hPD4UYenWdI8fr2H6ogtLqZrxeg3avjsej4/EaeLy6/6exxYPH61vcJdZuZUhqNENSoxmcGk1GUmSfNK+HQgz7s96On2HoGG1NGM31GC11GE3V6I0nW82qMRp9t/G0nf0CmqUjgYeBJQxQOpJzA7S3nvUpakzKqZqzcxhKZMJXzkYx2pr83XHe0kN4KwpQbFGYBk3AlDkBzTkCRev8pSHk5mEL0V8ptkg0WyRaYvYZjxmG4fs23lyL4WmD9laM9lZob8PwH7ditDWjN1VjNNXgKcvHaGs884VMFhRrJIo1HMUa4asRnH7bFolis3f61/D2ffO8pqoMz4hleEbsec/1eHWOlTdy5EQd+R0/2w6UA2A2qWQ5o/ybm8i0sv7N8Lb7vsyqJlBNX9m/e7I2bLQ1+ROn0dro+7elAb2lvuN3rM6XoFsawPCecR0lLAolMh41xomWNho1Mh4lMg4UzXd9dwtGewuGuwXcLf7bGAZqdGLH75Tdd52Tv2NhdtSw6G6tuKhYIzBl5GDKyPG9P10HRQmZ6aZSwxYXZCDH0PC4fTWDphpfjaCpuuMPVhO4m3zHbU0Ybb4/ZF9ZWwCwhKPY7Kj2BDRHFlriYN/KcOHRvfeGAlDT0Eb+iTqOnKgj72gNxysa0VSFMVlxXDbWyfgh8T26IttA/hyeZHjbfQmxuRa941+jrclXE7VG+L5I2iL8x1jCUVQVw9NGjNZE5dFCjPoy9Lpy9Poy9PpyjMZq4LS/t4pyKnlrJtBMoHt9n/mzJGDA1zoVFuUbHBoWhRoe7T9WwjvuC6CFKlRIDVuIEKaYLCjRyajRyV063/C2n1YDacRo8dVEwjU3jVWVvppIfRnu3WvB8DU5n1oZbjBaYjZqbIqv1uNu9dc0jPYW3213C0ZHLQT/8Zdvt/j6422RKLZI377m1sjTavwRqJHxqPEZ3VpGNtZuZfKIRCaPSATgeEUjm74oZdO+UnbnVxFmNXHZMDuXZaik2ppRNO2MloZz9Ut2l6F7wOv1xdHQMQwddN1/G0MHRe34UUBRUVTt1H2q6ktE7W3gOdnScuqY9jZfvNuaMNoaff+fJ4/bmjBaG8HjRo1JRo3PQIvP8P/b3bX1DU8bRnMdenMtRlOtLxE31/pu+xN0LbR1d8iTAmYbtLdweluRYrOjRCWiJQ9DjUpEsYSdiqfuwfB6QPeA7gWvxxe7Ti1HnY8xh4VMzfRiITVscUEkhoE7Yz12TxveyiL08ny85QV4ywt8+5B3lWpCsYb7alAdA3VODdgJg5ODclobTiWb1gbfH9/TKOExp5JNQkeyiUo8Y6qcYRjgdfsSWnsrhrsZvb4Cva4UvbaUpooTGPVl2PSWc5fbEub/Q6+Gx/iaRiMTOgYF+o6xRvj/+Pu6MurQa11EeKqpO34UvdaFXlOC0VTd9XgFyhzmS07WiFPdHtYI0My+QY+VxZ0GPSqR8R0JPB0UzfeFq70Vo+NL2OnHRmuDf5BUJ6rJVzsNj/HFquNYCY/uuN1xnzWyozvn9C8VTZ2OlbAoYtIH0UgUapRDNuu5QFLDFmIAUkxWTMnDIHmY/z69uRZveT5GXdmpEeyWMF+yODn45uT9F9CsaBgGeNwYrQ2+wXlVxXiritGrinEf/8Jf48dsQ7U7fLX80/r6OzWbnv5ewmMIj05GTZ6KNzKRgkYbn5+A0ipf0ohQ2ohUW7GrbTjwEIuHaE8b9sYTWI/vO7MbwWRFtSeAyYpeVwruZgBaTj4W40RzDkeNTvINTFKV02rSp2rPiqKCYfhq3obuqy2erI2frIkrGorZemqmgNmGYrL6jzHbfGMTutAyoDfXolcd88dUryrGU7TLFzfNgmKx+f4vzTYUiw0lIgbV7PQ1YYfHokacTMK+BH36F5fz0jpquufoXYl02GmRL979hiRsIUKYGh6DOmhSj11fUZSORNSREFNH+R8zPG5/TdFbVeyr7Zusp01/s4LJdlpys/mmzEUn+W6fZmzHD/jWQy+tbsZV1URpdTN5Vc24qpspK2/GqxtoKoxLszExTWNEnI7dqPeN7G+swmhvxTzkEt88+xgnjsFDqW41h+xCOWpHojWlj/XfZ3jbTzXFC9ENkrCFEGelmCxoCYPQEgYRzNno4TYT2SlRZKd07iv36jr5J+rZdaSS3Ucq+ctnvr7Z1IQYcoYMYfyoBLJTojotrWqKsqO09a8aoqyPLy6UJGwhREjQVJVh6TEMS4/hliuHUFbTzO6OpVXf3VLM2s1FWEwqYTYTNosJm0UjKsKKpoDNqmGzmAizaERFWIiOtBAdYSU6wkJMpIUwq0kGQIl+TxK2ECIkJcWGc+3UDK6dmkFzazt7C6opdNXT6vbQ6vbS6vbi0XVqm9z++1raPHi8Z/armzTVn7xTEiLISoki2xlFqiMCrYtrSAvR1yRhCyFCXrjNzLRRSUwbldTp/jNG2hsGLW0eahvd1DW5qWtqo+7kcaOb2sY2dhyqYP0eFwAWs8qgJDvZKdH+JB4XZZXauAhJkrCFEBcNRVEIt5kJt5lJSTj7NCXDMCivbaGwpJ6CknoKXPV8sP0Ynq2+mnlMpIVh6TG+FeDSY3DGh0sCFyFBErYQYkBRFIWk2HCSYsO5ZLRv8ZuTy64WlNRz+HgtB4/VsnW/b9lVe7jZl8A7+tfTEiNRJYGLPiAJWwgx4Jk031roWc4orp6U5q+FHyyu5dCxWg4W17L9YAUAUeFmLhmdzOVjnaQlfvUiF0IEmyRsIYT4ktNr4VfkpABQWdfCoWO17DxUyYfbj7Nu2zEyk+1cPtbJtFFJRIbJdC3RsyRhCyFEFyREh5EQHcZlY5w0NLvZklfGhr0uXnr/EK98dJjxQxKYPtbJmOw4GXkueoQkbCGE6CZ7uIVZk9OZNTmd4rIGNu71bXjy+cEKImwmwqwmVNW3LaOqgKoovr1GFAVVVUhNiPD3iyfGyiYZomskYQshRAAykuxkJNlZcOVg9uRXsSe/knaPgWEY6IaBbvhGpuu6gWH4BrjtLajisy9KAYiOsPgXjBmeHkOKI0IGtYmzkoQthBBBYNJUJg5zMHGY47znGoZBaXUzB4+dGtS27YBvVHqEzcSgZDvJcREkxYWRHB9Oclw4cVE2SeQDnCRsIYToZYqi4IyPwBkfwczxqRiGQVVdKweP+aaUHS9vZOMXLlrdXv9zLCaVxNhwkuPDfU3qadFkp0ZjNcsmIgOFJGwhhOhjiqKQEBNGQkwY08c6AV8tvL7J7dvZrLqZ0qpmSqubKS5rYPuBcgxAUxUGOe3+5vQhqTGE2+TP+sVK/meFECIEKYpCdKSV6EgrwzNiOz3W3OrhyIk6Dh6r4dCxWtZtPcY7m4tRgPSkSAanRhMVbiHcaiLcZvL/G2Y9eWyWxN4Pyf+YEEL0M+E2E+MGxzNucDwAbe1eCk7U+fvEN31R2qk5/Wxi7VZGZceTFh/O4NRoMpPsmE0yHS2UScIWQoh+zmrWGDkojpGD4vz3eXWdljYvzW0eWlo9NLe209zmpbmtncaWdopKGzhcXMPG3SUAmDSFzCQ7g1OjGZwaTUZSJAnRNplTHkIkYQshxEVIU1Uiw9RzrsDmcNg5XFhJ/ol68kvqyD9Rx8c7T7Bu2zHAl8STOga6OTtGqzvjI0iOCyfMKumjt0nEhRBiAIuJtDJpuINJw33T0U5uhHKioglXdROlVc2cqGhi56FKdOPUXuORYWYibCYiwnz94ZEd/eIRNjMRYWbs4Wac8eE44yKwWmQkezBIwhZCCOF3+kYop/N4dSpqW3BVNeOqaqKqvo3m1naaWj00NrdTVt1Mc6uH5lYPxpeuGR9lIyUhgpSEcFLiI0hJ8E1pk4Fv3SPREkIIcV4mTfXPHYevXhxG1w1a3B7qGt24qpopqWqipNL3s7+oBo9X95+bkRjJ2MHxjM2OZ3BqlPSXn4ckbCGEEEGjqoqvWdxmJiUhgkmnJXddN6ioa6Gksolj5Y3kFVbzzuZi3t5URJjVxKhBsYzN9iXwWLu1D99FaJKELYQQoleo6qltSycMdTBvehbNrR7yjlazt6CKvQVV/n3H0xyRpDkiCLOaOn40wjuObR3zyWPsVhzRtgGzeYokbCGEEH0m3GZi8ohEJo9IxDAMjlc08UVBFV8UVpNfUkdLm5eWNg9e/cs94z4RNhNZzigGOaPIctrJckYRE3lx1s67lLALCwtZtmwZtbW1xMTEsGrVKgYNGtTpHK/Xy8MPP8z69etRFIU777yTBQsWAPDEE0/w8ssvk5iYCMDEiRNZsWJFcN+JEEKIfk1RFNITI0lPjOS6SzL99xuGgduj09rm8c0r70jiFbUtFLrqKXQ18Pamo5wcxB5rt3YMnLOTmWxnUHLUOae39RddStgrVqxg0aJF5Obmsnr1apYvX84LL7zQ6Zy33nqL4uJi1q1bR21tLfPnz+fSSy8lLS0NgPnz53P//fcH/x0IIYS4qCmKgtWsYTVrRH+p9jxzQioAbW4vRWUNHHXVU1jaQGFJPTsOVfjPi4+ykZl8MoHbyUyyExVh6dX3EajzJuyqqiry8vJ4/vnnAZg7dy4rV66kurqauLhTq+qsXbuWBQsWoKoqcXFxzJo1i3fffZfvf//7PVd6IYQQArBaNP++4ic1tfpWdCsqbfAl89KGTkk8tqMPPDbKRqzdSqzdSpzdSqzddzs6woKqhk7/+HkTtsvlIikpCU3zTXzXNI3ExERcLlenhO1yuUhJSfHfdjqdlJaW+m+//fbbbNiwAYfDwQ9/+EMmTJgQzPchhBBCdBJhMzNqUByjTluytbnVQ3FH8j5W3kBVfRuFJfVsb2jrNOUMQFUUYu0W4qPDSIi2kRBtIz7aRkKUjfiYMOJ6eSR7rww6u/XWW7nrrrswm81s3LiRJUuWsHbtWmJjY8//5A7x8ZFBLZPDYQ/q9QYiiWHgJIaBkxgGZiDGLzM9lhlfuu/kdqZVda1U1rVQVdtCZV0rFTXNlNe0cOh4HZv3lXL62DdVgesvy+IHN43rlXKfN2E7nU7Kysrwer1omobX66W8vByn03nGeSUlJYwb5yv46TVuh+PUPLzp06fjdDo5fPgwU6dO7XJBq6oa0b9ilGB3ORx2KioagnKtgUpiGDiJYeAkhoGR+J3JblGxOyLIckSc8ZjHq1PT0EZlR1KvrG1lwojEoMVQVZVzVk7Pu6xMfHw8I0eOZM2aNQCsWbOGkSNHdmoOB5gzZw6vvvoquq5TXV3NBx98wOzZswEoKyvzn7d//35OnDhBVlbWBb0hIYQQoi+YNBVHTBgjM2OZMS6Fr1+RzZRRyb33+l056YEHHmDZsmU89dRTREVFsWrVKgAWL17M0qVLGTt2LLm5uezevZtrr70WgLvvvpv09HQAHnvsMfbt24eqqpjNZh599NFOtW4hhBBCnJtiGEZw2pl7mDSJhxaJYeAkhoGTGAZG4he4YMYw4CZxIYQQQvQ9SdhCCCFEPyAJWwghhOgHJGELIYQQ/UC/2a0r2MvDhdJyc/2VxDBwEsPASQwDI/ELXLBieL7r9JtR4kIIIcRAJk3iQgghRD8gCVsIIYToByRhCyGEEP2AJGwhhBCiH5CELYQQQvQDkrCFEEKIfkASthBCCNEPSMIWQggh+gFJ2EIIIUQ/MOASdmFhIQsXLmT27NksXLiQo0eP9nWRQtqqVau46qqrGD58OIcOHfLfL3HsupqaGhYvXszs2bO58cYbueeee6iurgYkjl21ZMkS5s2bx/z581m0aBH79+8HJH4X4o9//GOn32eJYdddddVVzJkzh9zcXHJzc1m/fj3QizE0Bpjbb7/deOONNwzDMIw33njDuP322/u4RKFt27ZtRklJiXHllVcaBw8e9N8vcey6mpoaY/Pmzf7bv/3tb42f//znhmFIHLuqvr7ef/z+++8b8+fPNwxD4tddX3zxhfG9733PmDlzpv/3y+xlJAAAIABJREFUWWLYdV/+O3hSb8VwQNWwq6qqyMvLY+7cuQDMnTuXvLw8f21HnGny5Mk4nc5O90kcuycmJoZp06b5b48fP56SkhKJYzfY7Xb/cWNjI4qiSPy6ye1289BDD7FixQoUxbfJhMQwcL0Zw36zW1cwuFwukpKS0DQNAE3TSExMxOVyERcX18el6z8kjhdO13X+8Y9/cNVVV0kcu+mXv/wlGzduxDAM/vznP0v8uunxxx9n3rx5pKen+++TGHbfT3/6UwzDYNKkSdx33329GsMBVcMWoq+tXLmS8PBwvvnNb/Z1UfqdRx55hE8++YR7772XRx99tK+L06/s3LmTvXv3smjRor4uSr/20ksv8eabb/L6669jGAYPPfTQ/9/encdHVd3/H3/dO2v2ZbInhCVsEUE2FUFcohSLKLS1tUX9drHYlqrfam21topV2xr7q2tRq1WrX622aisaqVWLVlFBBBQwrEkgIRmyb7Nklnvv748JUWQxySSZTObzfDzymMlw596TD0neueeec+6QHj+mAjs3N5f6+no0TQNA0zQaGhqO6PIVxyd17J/S0lL279/PPffcg6qqUsd+Wrp0KRs2bCAnJ0fq10sbN26ksrKSc845h5KSEg4ePMjll19OdXW11LAPDtXFarWybNkyNm/ePKQ/xzEV2A6Hg+LiYsrKygAoKyujuLhYun76SOrYd3fffTfbt29n1apVWK1WQOrYW263G6fT2fP52rVrSUlJkfr1wRVXXMG6detYu3Yta9euJScnh0cffZRFixZJDXvJ4/HQ2dkJgGEYrFmzhuLi4iH9PlQMwzAGfK/DWEVFBTfccAMdHR0kJydTWlrKuHHjIt2sYev222/ntddeo6mpibS0NFJTU3nllVekjn2wZ88eFi9ezJgxY7Db7QAUFBSwatUqqWMvNDU1sWLFCrxeL6qqkpKSwvXXX8+UKVOkfv1UUlLCQw89xMSJE6WGvVRTU8NVV12Fpmnouk5RURG/+tWvyMrKGrIaxlxgCyGEENEoprrEhRBCiGglgS2EEEJEAQlsIYQQIgpIYAshhBBRQAJbCCGEiAIS2EIIIUQUkMAWQgghooAEthBCCBEFJLCFEEKIKCCBLYQQQkQBCWwhhBAiCkhgCyGEEFFAAlsIIYSIAhLYQgghRBSQwBZCCCGigDnSDRBC9E9JSQlNTU2YTKae11599VWys7O56aab+OCDD9i/fz+//e1v+epXvxrBlgohBoIEthBR7KGHHmLu3LlHvD558mQWLVrE73//+wi06nDBYBCzWX7VCBEu6RIXYgS65JJLOO2007DZbF+47X//+18WLVrEjBkzmD9/Po8++mjPv73xxhssWbKEmTNncu655/L2228DUF9fzw9/+ENOOeUUFixYwN///vee99x///1cffXVXHfddcycOZN//vOfdHZ2cuONN3L66aczf/587r77bjRNG/gvXIgRTP7sFSLG/fKXv+See+5h9uzZtLe3c+DAAQC2bt3K9ddfz3333cdpp51GY2MjLpcLgJ/+9KeMHz+ed955h8rKSr773e8yatQoTjvtNAD+85//cO+993LnnXfi9/u59tprycjI4LXXXsPr9fKDH/yA3NxcvvnNb0bs6xYi2sgZthBR7Mc//jGzZ89m9uzZrFixol/7MJvN7N27F5fLRUpKClOmTAHg+eef52tf+xrz5s1DVVWys7MpKirC6XSyadMmrrvuOmw2G8XFxXz9619n9erVPfucPn065557Lqqq4nK5ePvtt7nxxhuJj4/H4XDwne98h1deeWVAaiBErJAzbCGi2KpVq456Dbsv7rvvPh588EH+8Ic/MGnSJH76058yY8YMnE4nZ5555hHbNzQ0kJKSQmJiYs9reXl5bN++vefznJycnud1dXUEg0FOP/30ntd0XSc3NzesdgsRaySwhYhx06ZN48EHHyQQCPD000/zk5/8hP/+97/k5uZSXV19xPZZWVm0t7fjcrl6QtvpdJKdnd2zjaIoPc9zcnKwWq2sX79eBp8JEQbpEhdiBPL7/fh8PgzDIBgM4vP50HX9qNu99NJLdHZ2YrFYSEhI6JkmdtFFF/GPf/yD999/H13Xqa+vp6KigtzcXGbMmMFdd92Fz+dj586dPP/881xwwQVHbUtWVhbz5s3jjjvuwOVyoes61dXVfPDBB4NaAyFGGglsIUagyy+/nGnTprFlyxZuuukmpk2bxsaNG4+67erVqykpKWHmzJk8++yz3HnnnUDozPt3v/sdv/3tb5k1axaXXnopdXV1ANx1113U1tYyf/58rrzySq666irmzZt3zPbceeedBAIBFi1axMknn8zVV19NY2PjwH/hQoxgimEYRqQbIYQQQojjkzNsIYQQIgpIYAshhBBRQAJbCCGEiAIS2EIIIUQUkMAWQgghokDUrGLQ2upG1wdmQLvDkUhzs2tA9hWrpIbhkxqGT2oYHqlf+AayhqqqkJaWcMx/j5rA1nVjwAL70P5EeKSG4ZMahk9qGB6pX/iGqobSJS6EEEJEAQlsIYQQIgpIYAshhBBRIGquYR+NpgVpbW0kGPT36X0NDepRb4Qgem+ga2g2W0lLy8RkiupvSSGEGDRR/duxtbURuz2ehIScw27n90XMZpVgUAI7HANZQ8MwcLs7aG1tJCND7pEshBBHE9WBHQz6+xzWYvhRFIWEhGRcrrZIN0UIIY5J03VaO300t3fR1N5Fc0cX0ydnU+iIH5LjR3VgAxLWI4T8PwohhgPDMGjp8FHd0ElNg4uGVm9PQLd2+tA/d4NLn2ZQeMa4IWlb1Ae2EEII0R9BTaeuyU11vYuaBhc13SHt7goCoACpSTYyUuxMGJVCRoodR7KdjJQ4HCl2HMk28nJTaWzsHJL2SmAPkOXLv00gECAYDFBTU83YsUUATJw4iRtvXNmrfbz44vP4fD4uvviSPh37yiuvoL6+noSE0Ao5hYWjufXW37Fnzy7uuquU3bt3cdpp87j99jv79kUJIUSUCQR19h5oY1tVC1V1HfiDGoGgHvrQQo/BnsdPz5atFpWCzEROnpzFqKxERmUnUZCZgN06fGJy+LQkyj3yyBMAOJ11fP/7l/GXv/z1iG2CwSBm87FLvnTpRf0+/k9+ch3z5s0/7LW0tHSuvPJa9uzZxYcfbuj3vsOhaRomkykixxZCxIaGVg/bKlvYXtnMzuo2fAENk6owJjeJxDgrFrMa+jCpPc/NJhWrRSUnPZ5RWYlkp8WjqsP70tyICex3tzlZt9XZq20VBYw+rCR3+rRc5k3t3+jliy66gMWLl7Bp00by8vK54ooV3HLLL3G73fj9fubOnceKFf8LwKOP/gmv18uVV/6ENWte5vXXXyUpKZnKygqSkhK5/fY7cTgyen3sjIxMMjIy2b+/6rjbdXV1cfvtK9m3rxKTyUxh4Whuu+0OAMrKVvPcc88CYLFYuPPOu0lPd7BmTRlPPfUEiqKQl1fAz39+I2lp6axZ8zJvvPEaaWmpVFVV8Ytf3ITfH+Chh+7H7XYD8P3v/5C5c0/vTzmFECOYpuvsqm7jw12NbNndiNcXJN5uJiHOQoIt9BhvN5Ngt5BgN9PhDrCtqpmGVi8Amal25k7NYepYB5NHpw6rs+OBMLK+mmGqqamJ++//EwA+n4/S0ruJj48nGAxy7bVXsn79e8yZM/eI9+3YUc4TTzxDdnYOpaW38/zzf+MHP/jxUY9xzz3/j0ceeRCAr3/9m5x//oW9bt+GDe/T2dnJU089B0BHRwcAmzd/yP/93+M88MCfcTgy8Hg8mEwmKiv38sAD9/HnPz9FRkYGjzzyIHff/XtuvfV3AGzb9hF/+csz5OcX0NnZydVX/4Df//4+MjIyaGpqYvny/+HJJ/9GUlJS74sohBiRgprOjv2tfLizgS17mnB5A1gtKtPGOXCk2HF3BXF7A3i6gjS2deHxBXB7g/gCGlazyuTRaSyYPYoTx6WTnTY0o7UjZcQE9rypvT8LHup52Oedd37Pc13XeeCBe9m2bStg0NzczJ49u48a2NOmnUR2dg4AU6acyMaNx+7WPlqXeG+NHz+B6up9/OEPpcyYMavn7Pf999/lvPPO7zmrj48P/TBs3vwhp512OhkZodeXLPkq3/nOsp79TZ06nfz8AgC2b/8Yp7OO6667uuffFUWhtraGyZNP6Fd7hRDRzeUNsKemjc27G9mypwmPL4jdamL6+AxmTcrkxHEObJbjX0oLaqHf4WZT7CzYOWICeziLj4/ref63vz1NZ2cHDz/8F2w2G6Wlv8Hv9x31fVartee5qprQNG1Q2pefX8DTTz/Hhx9uZP36d3n44VU88cSzGMe4bmAYR07D+uynn/16DQOKiiawatUjg9J2IcTwpusGBxpdVNR1UFHbTkVtO/XdXdjxNjPTJ2Qwe1IWU8amYTH3frxLLAX1IRLYQ6yzsxOHIwObzUZjYwPr1v2XpUu/FtE2NTTUk5ycwhlnnMUpp8xh6dLz6OzsYN68+dxxx20sWfJV0tMdeDwezGYzs2adzF//+gTNzU04HBm8/PKLzJ59ylH3feKJ0zhwoJrNmz9k5szZAOzY8QmTJ58gc6+FGEECQZ3Wzi6aO3y0dHRxsMVDZV0Hlc4OfP7QyUZyvIWi/BTmn5RHUV4yRfkpMRm8/SWBPcS+/vVvctNN1/Pd7y4jKyubWbNOHrRjOZ11rFjxfbq6uvD7fXzlK4u4/PIrWLx46WHbVVTs5aGH/giArmtceul3egasXXbZd/jJT1agKCpWq4XS0rsZN66IH/3oKq655sfdg87y+dnPbjxqG5KTk7njjrtYtepe7r33DwSDAfLy8iktvVsCW4go0uUP0tQWWkCksT20mEhLx6cB3e4+/J4OqqIwKiuReSfmUJSfQlF+Cpkpdvm5D4NiHKvfc5hpbnYdcZPwgwf3k5Mzus/7krXEwzcYNezv/2e0ysxMGrIFF0YqqWHf6LpBlz+I16fhC2gkJNqpb+zEH9DwBXT8QQ1/QMMf0Glz+7oD2ktjWxcub+CwfVktKo5kO+nJoQVEQo+ffp6WZMdiHvlnzwP5PaiqCg5H4jH/Pewz7KqqKm644Qba2tpITU2ltLSUMWPGHLZNc3Mzv/jFL3A6nQQCAebMmcOvfvWr485JFkII8cV8AY2m9i6a2ryhs982Ly2dPrxdAbx+Da8v2P0RCuneMqkKjhQ7mSl2Zk3KJCPFTmZqHBkpcWSk2EmKt8jZ8hALOzFXrlzJsmXLWLJkCatXr+bmm2/mySefPGybhx56iKKiIh5++GECgQDLli3jtddeY9GiReEeXgghYkKH28/++k72HeykrslNU5uXxvYuOj7XFW0xq6Qn20m0m4mzmUlPshFnM3/6YTVht5mxWUxkOhLwev3YzCasFhWrxYTVHHqMt5mH/UIisSaswG5ubqa8vJzHH38cgMWLF3PbbbfR0tJCenp6z3aKouB2u9F1Hb/fTyAQIDs7O7yWCyHECNXm8rH/YGfoozukWzs/nU3iSLaTmWpnWpGDzENnvqlxZKbYSU6w9vrMVy4pRJewAtvpdJKdnd2z9KTJZCIrKwun03lYYK9YsYKrrrqK008/Ha/XyyWXXMKsWbPCa7kQQkQ5n1+jtslNbaOLA41uaptCj4fOmhUgOz2eiaNSGZ2dxJicJAqzE4m3WyLbcBERQ3IR+dVXX2XSpEk88cQTuN1uli9fzquvvsp5553X630c7UJ8Q4OKuZ+DGvr7PvGpga6hqqpkZsbW6mex9vUOhuFaQ03T6XD7aXP5aHf5aHP5aXf5aO3o4kCDi/0HO6hv8fQsk2y1mCjMSeLkE7IZm5fC+IJUxuYlD3o4D9f6RZOhqmFYgZ2bm0t9fX3PDR40TaOhoYHc3MNXHHvqqaf47W9/i6qqJCUlUVJSwoYNG/oU2EcbJa7rer9GKsso8fANRg11XY+p7jnpjgzfcKmhpuvsrmlny+5GdlS30u7y4/YGONoUHJOqkJUWR0FmInNOyCY/I5GCrAQyU+KOuGbs7uzC3dk1aO0eLvWLZlEzStzhcFBcXExZWRlLliyhrKyM4uLiw7rDAQoKCnj77beZNm0afr+f999/nwULFoRz6GHn2muv4owzzjpsERTDMPjGN5bwy1/ewvTpM4/6vt/85hYmTy7ma1+7+Li311yz5mXee++do94iU26vKcTQ6/IH2V7ZwpY9jWytaMbdFcRsUpk8OpWJBakkxVtITrCSHG/99HmClXibWUZXi34Ju0v8lltu4YYbbuCBBx4gOTmZ0tJSAJYvX87VV1/N1KlTufHGG1m5ciUXXHABmqZx6qmn8o1vfCPsxg8n559/IX/729OHBfaWLZswmUzHDOvPk9trCjF8ebqCNLV7qXJ2sGVPE+X7WglqOgl2MyeNz2DGhExOHJuOzToyv98NwwC/B93dguFqBUNDTStAScro8x8gRtCP3tkIioJitqFY7GC2oZh6H0mGEerhU5Shu7xpaEEMnxvD7wafB8PnJmidBNiH5PhhB3ZRURHPPffcEa8/8sina0cXFhb2jCQfqc444yzuuusOqqoqGTt2HACvvPISixZdQEXFXv7whzvo6vLi9/u58MKv8I1vLDtiH5+9vWYgEODuu+9ky5ZNZGZmUVg4ps9tkttrCtF7hmHQ3N5FbZM7NK+53UtTW2hVr6a2Ljy+YM+2GSl2zp6Rz8yJGYwvSMGkjowxMYYWRG+tRW/aj97RgO5uxXC39DwS9B/5JosdNb0AU/ooVMeo0GN6AYo1DsPnRm9zorfWobXV9Tw3OpvgaBcMVBNY7N0hbgMUDD0IWhD0IIaugRYATQNDC93EwBqPYo1HsXU/WuNRbAnQ83kcijUOLKFHxWIPfW6NQzHbQgHs7cDwtqN7OzA87RhdnaFHbweGz4XRHc4Ej7zvQ9P4WZhKrhrw/4ujGTErlwR2v0tg19u92lZRlGPe2OJoLJPOwDJx3vG3sVhYsOA8/vWvl1mx4n/xeNy8885/+eEPryQhIYF77nkAq9WKx+Phiiu+zSmnnMaYMWOPub/Vq1/A6azj//7v7wSDQX784+VHjA34LLm9phB9E9R0qutd7K1tZ++BNvbUttPu+jSQbGYYmxxkcoKP3EIvGWY3KYaLeFOAuJR01IQGFHcqxoFUtPhUlPhUFHsySnd4G0F/KAx8nu4zMnfPL34j0AWBLoxAF4bfC0Ff9/MuCHZh6DoKgKKGQunQB5/5XDWFzi4VNRR0aui5oqhgMqPEJaPEp6HGp6AkpKHEp6DGp4EtIfQ7MOCj60Ad/r3l6E3VaE370VsPgN69uIqiht6XkIbJUYhSeBJqQjpKYhpqQuiyp9ZyAL25Br2lhkDFetjx5qcFtiWAz/3p5yYzakoOpsyxqBPmoqbmAApG0AeB0NcfqkPog2D3tXvVDKoZxWQKPTeZUVRz6Gs2dAy/p7vGHvB50Nudn35+tD8wesMShxKfjBqXgpqUCRkJKLZDH6E/CBRr6HnWhMk0dwS/eJ8DYMQE9nBw/vlLuO66q7jiih/zn/+8zrRpJ5GZmUVLSzN//OMd7N27G0VRaWpqZO/e3ccN7M2bN/HlLy/GbDZjNptZuPDLbN360TG3l9trCnFs+qGz50Y3VQeaqTtwkNbGRuIMD4mKl7y4IKek6mTk+EkyPFh8LSjettDt5tyEPhQFJT4NxRqH1lpFsOsoA40UFcWWEAofLXDkv3+Wag51BVvtoUeLPRQGiendYWQARujRMELh1POaDrre/ahhBIPdn2sYho4RDGB428HvPfpx7YkY3nZc3Scuij0J1VGIdepC1IzRmDJGoyRl9fzxcSym7PE9zw3DCJ2NN9egtdRguJpRkrIwpeWipuahJGV+4f4GmqEFQ38Y+b0YAW/oj6OAF8PfFQr0gA9s8ahxKaE/cA59mK1fvPNuqi0OGJqBeyMmsC0T533hWfAhgzVKfMKEiTgcGWzY8D5r1rzU0+39pz+tIj3dwWOPPY3ZbOaaa36M33/8v/yGcol3ub2mGK4MPYjeXo/ecgCtuQZ/RzOYbajWOFRrHC1pyfgCKmp3V6dhMtPR0kp7UxOe9hYCnW0Y3g7MATcJioc8pYsitTtIPz8YN2BHMSejxqeiZJyAmpSBmpSBkpSBmuhASUg/7BqroQUxvO0YnjZ0TxuGuw3D04bR5eoO38+cjdkSDu+qtcT16Xptv+sX9GF42g9vn6cN3duBmuggragYlyUz9LWFORBOURSURAdqogPz6OkD9BWERzGZwZSIYj/2yOtoMmICe7g4//wLeeyxh6mvd3L66WcC4HJ1UlQ0AbPZTGXlXj7++CMWLDj+lLbZs0/m1VfXUFKyAE0L8vrrr5KdnTMobZbba4pwGX5v6Lqf39vdxdnV3a352S5OH6gqiskKZmvoLMZs634MfW74XKFu1pYDBJoOQLsTxQh10WqGQqcRh4UgdiWASTFoO0pbrEBm93OPYcNnSkBLTMQUn4WelIopPQNzUiqqPRkl/rNnVbY+fc2KyYyS6IBEB8N1mJlitqEkZ6EmZx313xMyk/DItK6oIYE9wBYs+DKrVt3HkiVfxWIJLXjw7W9fzm233cxrr/2L/Px8pk+f8YX7ufDCr7J3714uu+wbZGVlM336LJzO2j61RW6vGf2MoB+tbgd6ay1q5lhMWeP6HCxhHT/Qhe5qwXC3YLha0L3tocE4ntCZq+5pw/C0H3UwTjjajERqAyk4tcnU62noKfmk5hWSnZkCgKYb6EE/CRYDT0cnSrALRetCNYIkpaSSkZ1Ndl4m2Z/p7REi2sntNUW/yO01w3esBRcMn5tgzVaC+zYTrNkGgc8snKGYUDNHY8qZ2P0xAdXet8F7hqHDoTPiLlf3Rye6pz10DdLVHApnd8vhg4YOscaFuo3jUkIDreJTQgOb4pJ7Rt6GrsnaQn9cdI/6xWwBw0AP+Ghs6qC6roW6+lYONrTR2tqBmSBdhgVffDb5eZmMy0umKC+FwuxErJajn8PKwh/hkfqFL2oWThFCDAzd3Upw/xaCVZvQ6naCoaHEJWMpmoN5zEzUjNHoTfvQDu5BO7ibwPY3CGx9FQA1NQ81qyh0vU4PhgbaaIHQY/eUGEMPhuaNdnWGpqcYx/hjy5aAmpiOkujAkjMBJTG9e2SwI/QYn9KnATmHNLR62LDDyZ6aNirrOnqmSNmtJsbm5jGxaDLjcpMZm5dMauLQ9SAIEU0ksIUYAoahY7jb0F1NGJ1N6J2NNAba8TQ60TubMDobAVCSs7FM/RKWsbNQs8YdtiiEWngS5sKTQvsL+tGa9qMd3BUK8ZqPQyP8TJZPp72YDk2HMYfOeuNTUeyJKPak0KPtM8/tSaHruJaBC0uXN8DGHfW8/0k9e2vbUYD8zERmT87qPntOJteRILdwFKKXJLCFGECG3xtaHOIzi0TobaFQRj98rqaWmIaR4MCUVYQ6+UzMY2aEpr/04tq+YrZizpmAOWfCYH0p/RII6ny8t4n3PznI1opmNN0gPyOBi84qYs4J2aQnD82KUEKMRFEf2IZhyOClESBKhlIA3aHc2YTe2YDR0RRaEardid7mxHC3frqhakJNyUZNy8c8ZmZoelBSZmiqUKKDrFzHiLh+2O72s7umjU+qmvlwZyMeX5CUBCvnzCpg7ok5jMpKlJ9RIQZAVAe22WzF7e4gISFZfiFEMcMwcLs7MPfj2mi/jqcH0Rv3EXTuQnPuwnA1d08rsnUPlDo03cgWel1RugM61HVtfH7BDEscamoOprwTUNNyUVNzMaXmoSRnhrqmR5jWTh+7qlvZVdPGruo2DrZ4ALBZTMycmMFpU3IoHpM2YpbrFGK4iOrfJmlpmbS2NuJyHW025rGpqoquyyjxcAx0Dc1mK2lpmV+84efonjbQ9Z65vJgsR/zxZmgBtIZKtO6A1ur39CxZqKbmoabmYAT9EPSHpisFfKFlJYPdc4d1AyXJgZqUiWnMLJTkzNCZcvfjoaUeo1GH28/Wima2VjbT3N6FxayGPkxqz3OrOXTf+S6fxu6aNhraQqtnxdlMTChIZf60XCYWpjI6OwmzSUJaiMES1YFtMpnJyDj2+trHIlMZwhfJGhpdLgKVHxDY/S56Q8Xn/lX5zKIcoUe9szF08wBATS/AMmk+ptzJmHInocYlf/HxRtBlF8MwqK538fHeJj6uaGafswMDSE20kp+ZiKbpdPmCdAZ1AppOIPjph8mkUJSXwtkz85lUmEphVpIMGBNiCEV1YIvYYWhBgjUfE9z9HsHqj0DXUNPysZ58EUpcUujsOOgLPQa6H4N+CPowjZqGKXcS5pyJ/VqiMJrDWtcNGtq8VNd3Ur6vhY8rmml3+VGAsXnJLJk/lpOKMijMluvMQgx3EthiWDIMAwJd6K21BPa8T7BiA4bPFZqbPOVcLBPmojoKJWQ+o9Pj50CDi5pGNwcaXRxocFHX5MbfvcBNnM3ElLEOTipyMHWcg+SEoRkzIIQYGBLYIiIMvxetaV9oZLWnPXQTBW9HaLWt7nvT9twaz2TBPHoGlonzMBWciKIO15Wbh45hGNQ1e9hd3crO6jb2HGij7TO3hkyKtzAqK5GzZuRTkJlIQVYCBZmJco1ZiCgmgS0GnaHr6G11aA0V6A0VaA2V6K213bcPBFBCi3fEpYTuQZtcFHoel4KamI65cBqKNT6iX0Ok6YZBXaO7e2R2aIR2pyd016m0JBuTCtMYk5PUHc6JpMjZsxAjjgS2GHCGrqE1VKAd2B5ahaux6tP1sG0JmLLGYR0zK7RgiGNUaIUtOWvu4Qto1DWFurVrG93UNrrYX+/C5Q0FtCPZxoljHUwuTGVSYSqZqXFyaUCIGCCBLQaE3tlIsGY72oHtBGvLIeAFRUF1jMYyYR6mrHGYsopQUrJjPlwMw8DdFcRzsIOqmlbaXT4aWr3XgR14AAAeQUlEQVTUdl97bmj1cqjvwWJWyctIYPqEDCYWpDK5MJWMVLkDlRCxSAJb9Ith6AQPbCe4/yOCB7ZjtB8ECN00ougUTAUnYs4/AcWWEOGWRlZLRxdvf1xHbaObNrePdpefNpefoHb4HHYFyEqLoyAzkVNPyO7p2s5KjZOpU0IIQAJb9JGhBwnuXc+Bf7xKoOkAmKyY8iZjPqEE06gTUVNyY/4MGqCitp3XP6zhw52NGIZBjiOe1EQb4wtSSE2wkZJoZVRuCqquk5pkIy3Jhu0Yt5AUQgiQwBa9ZAR8BHb+F//WVzHcLVizCrGffQXmsbP7dbvFkSio6Xy4q4HXNx6gytlBnM3EubMLOGdWAZlH6caWBXyEEH0hgS2Oy+hy4f/kDQLb38DwuTDlTMQ6/9vkzJxHU5Mr0s0bFjo9ft76qI43Nx+gzeUnOy2OSxZMZO6JOcTZ5EdMCDEw5LeJOILu7UBv2kewZhuBnW+HVgsrnI51+vk9t3OM5W5vwzCobXSzraqZ7ZUt7K5pQ9MNpoxJ49vnTWZqkQM1husjhBgcEtgxzDAMDHcLelM1WtM+tKb96M37P71FpKJiHj8H60mLMKUXRLaxEebpClC+r5Vtlc1sr2qhtdMHQEFmAgtOHsW8qbnkZ8T2ADshxOCSwI5BhhbAv2k1gZ3//fRWkYoSui1k7mRMGWNQM0ZjyiiM6QVLXN4AG8rr2bCjnsraDnTDIM5mZsrYdKaOTWfK2HTSk+2RbqYQIkZIYMcYrXEfXW89gt5ai3nsbEx5xZgyRocWMDHbIt28iNN1g/L9Lazb6mTz7kaCmkFBZiKLThvN1HHpjMtLlvs8CyEiQgI7RhhaEP+Wl/BvKUOJSybuvGsxF06LdLOGjYY2L+9udfLudictHT4S7GbOnJ7P6VNzGZ2TFOnmCSGEBHYs0JprQmfVzdWYJ8zFPveSEb+gicsbYOPOBjbuqKfTG8BqVrGaTVgsoUerRe157UCji53VbSjAlHHpXFwygenjM7CY5UxaCDF8SGCPYIau4f/oFfybV6PYErB/6WosY2ZGulmDxhfQ+HhvE+s/qWdbZTOabpDriCfXkYA/qBEI6Li9AVqDPgIBHV/3a0kJVr5yxjjmnZgj16SFEMOWBPYIpbXW0vXWn9EbqzCPOwXb6Zeh2kde166m6+zY38r6T+rZtLsRn18jNdHKubMLmHNCDoXZiTE9BU0IMXJIYI8wRpcL3+bVBD5Zi2KNw37OCixFp0S6WWHRdYPWTh+NbV4a2700tnXR1Oalsc3LwRYP7q4gcTYzp0zOYs6UHCaNSpX1t4UQI44E9ghh6EEC5W/i2/Qi+D1YJp2J9eSvosYlR7pp/bZpVyP/eLuChlYvmm70vK4qCunJNjJT45g1KYup49KZVuTAYpa1uIUQI5cE9ggQrN6Kb/0z6G1OTPknYJvzLUyOUZFuVr8ZhkHZ+/v559uVFGYlsvCUQjJT7WSkxpGZGkd6kg2zSQaECSFiiwR2FNNaa/GtfxatZhtKSjZxX/pfTKOnR/U1W39A4/F/7WRDeT1zpmTz3S9PljNnIYRAAjsqGbqOb/2zBD55Ayw2bHO+hWXKOSim6P7vbHP5uP+FbVQ5O/jameNYNGd0VP/xIYQQAyns3/BVVVXccMMNtLW1kZqaSmlpKWPGjDliuzVr1vDggw9iGAaKovD444+TkZER7uFjjmEY+N57ikD5WizFZ4euU4+A0d/7D3Zy3wtb8XQFufKrU5k5MTPSTRJCiGEl7MBeuXIly5YtY8mSJaxevZqbb76ZJ5988rBttm3bxh//+EeeeOIJMjMz6ezsxGqVeyj3R2Drq6GwnvZl7HMujnRzBsSHOxv4c1k5ifEWfnHpTAqzo/8PECGEGGhhjdxpbm6mvLycxYsXA7B48WLKy8tpaWk5bLu//OUvfO973yMzM3TWlJSUhM0m61b3VaDiA3wb/haaV33q1yPdnLAZhsFL66p44MXtjMpO5KZvnyxhLYQQxxDWGbbT6SQ7OxuTKTQoyGQykZWVhdPpJD09vWe7iooKCgoKuOSSS/B4PCxYsIAf/ehHcn2yD4LOXXS9+TCmnInYz/o+ijL8R0lX1nVQ5ezA5Q3g8gZwewO4urofuz+8Po3TpuTwnS9PksFlQghxHEMySknTNHbt2sXjjz+O3+/n+9//Pnl5eSxdurTX+3A4Ege0TZmZ0XMm52+upe71+7GkZZG37EZMccOj7ceqodcX5IlXynnl3aqe1xLsZpISrCTGW0lLjqMwJ4WkBAvjC1IpmT0qZv94i6bvw+FKahgeqV/4hqqGYQV2bm4u9fX1aJqGyWRC0zQaGhrIzc09bLu8vDzOO+88rFYrVquVc845h61bt/YpsJubXeifWTwjHJmZSTQ2dg7Ivgab7mnHs/p2UFSsC35CiwtwRb7tx6rhrupWHluzg6a2LhbMHsWiOYUkxluOe0vKpibXYDZ12Iqm78PhSmoYHqlf+AayhqqqHPfkNKx+VYfDQXFxMWVlZQCUlZVRXFx8WHc4hK5tr1u3DsMwCAQCrF+/nsmTJ4dz6JhgBHx4/30PhqeduIU/QU3OinSTjskX0PjrG7u5869bAPj5shl869wJpCTa5P7RQggxAMLuEr/lllu44YYbeOCBB0hOTqa0tBSA5cuXc/XVVzN16lTOP/98tm/fzqJFi1BVldNPP52LLroo7MaPZIau4/3Pg+hN+4hbcDWmrHGRbtIx7T3QzqOvlFPf6uWcmQVcdFYRNqtcjxZCiIGkGIYxMP3MgyyWusQNw8D37v8RKF+Lbd6lWKecG+kmHSEzM4naujZefKeKf39QjSPFzncXFVM8Oi3STYsaw/37MBpIDcMj9QvfUHaJR/fSWCNUsGpjz1zr4RbW7q4Au6vb2P/uPt7bWkdTexdnTc/j62ePJ84m305CCDFY5DfsMGPoQXwbX0BNK8B2SuTnWnu6guw+0MbO/a3srG6lpt6FAVjNKkX5KfzPeZM4cawj0s0UQogRTwJ7mAnsWofRXo994f+iRHCw1q7qVv7+ZgX7DnZgGGA2qYzPT+bC08cyuTCVU0/Kp63VE7H2CSFErJHAHkaMoA//phdRs8djKpwemTYYBv/+oIbn36rAkWJj8WljmDw6jaK8ZKyWTweSySInQggxtCSwhxH/9v9geNqwnxOZVeC8viCPr9nBh7samTUxk++dXyzXpYUQYpiQ38bDhOFz4/+oDNOoaZhzJw358Wub3Dzwz20cbPHw9bOLOO+UwphdfUwIIYYjCexhwv/xv8DvwXbK0M9P/2BHPY+v2YnNovKzb85gskzNEkKIYUcCexjQPW34t72GefwcTI7CITtuUNN57s0KXv+whvH5Kfxo6YmkJcld1IQQYjiSwB4G/JtfAl3DNvurQ3bM1k4fD63ezp4D7Zw7q4BvlIzHbJIlRIUQYriSwI4wvb2ewI7/Yik+a8jWCt+yp5HH1+wkENT5wYVTOPWE7CE5rhBCiP6TwI4w34f/BJMJ68wLBv1Y/oDG397cy5ubaynMTuQHF04h15Ew6McVQggRPgnsCNKa9hOsWI91xgWo8amDeqyaBhd/eukT6prcLDxlFF89owiLWbrAhRAiWkhgR5Bv4wtgS8B60pcH7RiGYfDGpgM892YFCXYz1158kiwlKoQQUUgCO0KCdTvRarZiO/ViFGv8oByjw+3nsTU72FrRzElFDr57fjHJ8dZBOZYQQojBJYEdAYZh4Nv4PEpCGpYp5wzKMbZWNPPYmh14uoJcsmAiJTPzZSEUIYSIYhLYEaDt/wi9fi+2M76LYh7YM15PV4Bn/7OXdduc5GcmcN03p1OQeez7qwohhIgOEthDzDAMfJteREnOxjLx9AHd90d7m3jy1Z10uAOcf9poLpw3Rm7SIYQQI4QE9hDTaraiN+/HfublKOrAhKnLG+CZN3bz/if15GcmcNXXpjE2N3lA9i2EEGJ4kMAeQoZh4Nv8EkqiA/OE0wZkn5t3N/Lkv3fh9ga4cN4YFs8dIyuWCSHECCSBPYS0uh3oDRXYTv8fFDW80nd6/Dz9+m4+2NHAqKxErv3GSRRmJw1QS4UQQgw3EthDyL/5JZT41LCvXVfWdXDfC1txewMsnT+WRXNGy1m1EEKMcBLYQyR4cDeacye205aFNTJ8y+5G/vTSJyQnWPnpd05mVJaMABdCiFgggT1E/JtfQrEnYSk+s9/7eOPDGp55Yw9jcpO5+qJppCTIIihCCBErJLCHgNZQiXZgO9ZTvo5i7vv9pnXD4O9r9/LaxhpmTMjgigunYLPIdC0hhIglEthDwL/l5dCa4SeU9P29AY1HXi5n0+5Gzp1VwDfPmYCqyoplQggRaySwB5nWXENw/xass76CYo3r03s7PH7uf34rlXUdfOucCSw4edQgtVIIIcRwJ4E9yPxbXgaLHeuJ5/bpfQdbPNzz949pdflY8ZWpzJqUOUgtFEIIEQ0ksAeR1lZHsHIj1unno9gSev2+KmcHd/3tI1RV4efLZlCUlzKIrRRCCBENYm7yrtZcTfPrj2MY+qAfy7/lFTBbsEz9Uq/fs/9gJ3949iPibGZ+edksCWshhBBADAa23tFA+wdlaAe2D/pxgnvfx1J8Nmpc79b13n+wk//37BbibGZ+vmwGWWmDc59sIYQQ0SfmAttcOB1TQgr+T9YO6nH8H70Cqop12nm92r66PhTWdquJny+bQUZK3waoCSGEGNliLrAVk5mk6eeiVX+M3tk0KMfQXc0Edq/DMukM1IS0L9y+psHF/3v2I2xWEz9bNpPMVAlrIYQQh4u5wAZInrEAFAjseGvA921oQXwbngMDrNPP/8LtDzS6+P0zW7CYVX7+rRlkSVgLIYQ4ipgMbHNKJubC6QR2vY2hBQZsv1prLZ7VtxGsWI91+iLURMdxt69tcvP7Z7ZgNimhsJZr1kIIIY4hJgMbwHJCCYa3g2DVprD3ZRg6/q3/xvOPlRiuFuwLrsJ28teO+5667rBWFYWffWsG2ekS1kIIIY4tZudhmwqmoCRnEShfi2X8nH7vR3c10/XWn9HqdmAqnI79jO+ixh9/KlZ9i4ffP7MFgJ8vm0Guo/dztIUQQsSmmA1sRVGxnnA2vvV/Q2uuweTo27KfhmEQ3PMeXe8+BRjYzvgulklnoCjHX+c7qOk8tPoTNN3g+ktmSlgLIYTolbC7xKuqqrj44otZuHAhF198Mfv27TvmtpWVlZx00kmUlpaGe9gBYZk4H0wWAuV9m+Kld3XS9fof6XrrEUyOUSR87Vask8/8wrAG+NeGavbXd/Lt8yaRnyFhLYQQonfCDuyVK1eybNky/v3vf7Ns2TJuvvnmo26naRorV67k3HP7tqb2YFLsiZiLTiWw5z0Mv7dX79FdLXheuJlg9UdYT/kGcYtvQE3O6tV7DzS6eGldFSdPzmLWpN69RwghhIAwA7u5uZny8nIWL14MwOLFiykvL6elpeWIbR9++GHOOussxowZE84hB5z1hBII+gjsee8LtzWCfryv34/h9xK/5FfYpi9CUXtXQk3XefSVHcTZzFzypYnhNlsIIUSMCesattPpJDs7G5PJBIDJZCIrKwun00l6enrPdjt37mTdunU8+eSTPPDAA/06lsORGE5Tj5CZmQSAkTGN2vVF6LvfIuPMJcfs1jYMg8aX/4jeWEX2RT8nYdK0Ph3vuf/sZv/BTq7/n9kUjT7+dK9ocaiGov+khuGTGoZH6he+oarhoA86CwQC3HTTTfzud7/rCfb+aG52oevGgLQpMzOJxsbOns/ViWfS9fZjHNy2CXPupKO+x7/t3/i2vYV11lI86Sfg+cz7v0hto4u//nsnsydlMikv+bBjR6vP11D0ndQwfFLD8Ej9wjeQNVRV5bgnp2EFdm5uLvX19WiahslkQtM0GhoayM3N7dmmsbGR6upqrrjiCgA6OjowDAOXy8Vtt90WzuEHjHn8qbD+WQKf/OeogR088Am+9c9iHjML68wL+7RvTdd5bM0O7FYzl37p6H8MCCGEEF8krMB2OBwUFxdTVlbGkiVLKCsro7i4+LDu8Ly8PDZs2NDz+f3334/H4+H6668P59ADSjHbsEyaT2D7G+ieNtT41J5/0zsa8P7nAdTUPOxnfR9F6dtl/9c+qKHK2ckPl0whOcE60E0XQggRI8IeJX7LLbfw1FNPsXDhQp566il+/etfA7B8+XK2bdsWdgOHirX4bDA0Ajvf7nnNCHTh/fd9AMQt/F8Ua9/W+a5rcvPPd6qYNTGTkyfLqHAhhBD9F/Y17KKiIp577rkjXn/kkUeOuv1VV10V7iEHhZqagyl/CoEdb4Vu2qEodL35CHpbLXFf/mmvp24dousGj63Zgc2icunCSb2aoy2EEEIcS8yuJX40lhNKMNwtBKs/xr/lZYL7NmE79WLMBSf2eV+vbayhsq6DSxZMJEW6woUQQoQpZpcmPRrz6OkoCWn43n8Go7MR84S5WKYu7PN+nM1u/vF2JTMmZHDqCdmD0FIhhBCxRs6wP0NRTViKz8LobETNHIt9/nf61ZX919d3Y7OoXCZd4UIIIQaInGF/jnXKuRh+L9apC1HMfe/Krmty88m+Vr525jhSE22D0EIhhBCxSAL7cxRbAvY53+z3+9/cUovZpDB/Wt4AtkoIIUSsky7xAdTlD/LediezJ2fJnGshhBADSgJ7AK0vr8fr0yiZURDppgghhBhhJLAHiGEYrN1Uy6isRIrykyPdHCGEECOMBPYAqajt4ECji7Nn5svIcCGEEANOAnuArN1ygDibiTky71oIIcQgkMAeAB1uPx/ubGDuibnYrTLwXgghxMCTwB4A72ytI6gZnD0jP9JNEUIIMUJJYIdJ1w3e2lLH5MJU8jISIt0cIYQQI5QEdpi2VjbT3NFFyUyZyiWEEGLwSGCH6c3NtaQkWpk+ISPSTRFCCDGCSWCHoaHVw/bKZs48KQ+zSUophBBi8EjKhOGtj+pQFIUzp8tgMyGEEINLAruf/AGNdz6uY8bEDNKS5K5cQgghBpcEdj9t3NmAuytIiUzlEkIIMQQksPvpzS215DrimTw6LdJNEUIIEQMksPth38EOKus6OHuGrBsuhBBiaEhg98Obm2uxWlTmnpgb6aYIIYSIERLYfeQLaGwor2fOCTnE22XdcCGEEENDAruPdu5vxR/UOXlyVqSbIoQQIoZIYPfRtspmrBaViaNSI90UIYQQMUQCuw8Mw2BbZTPFhWlYzFI6IYQQQ0dSpw/qW700tnUxtcgR6aYIIYSIMRLYfbCtohmAqeMksIUQQgwtCew+2FrZTE56PJmpcZFuihBCiBgjgd1LvoDGruo2pkl3uBBCiAiQwO6lnftbCWq6dIcLIYSICAnsXpLpXEIIISJJArsXZDqXEEKISJP06QWZziWEECLSJLB7QaZzCSGEiDQJ7F6Q6VxCCCEiLezbTVVVVXHDDTfQ1tZGamoqpaWljBkz5rBtVq1axZo1azCZTJjNZq655hrmz58f7qGHxKHpXGfPyI90U4QQQsSwsAN75cqVLFu2jCVLlrB69WpuvvlmnnzyycO2mTZtGt/73veIi4tj586dXHrppaxbtw673R7u4Qddz3SuovRIN0UIIUQMC6tLvLm5mfLychYvXgzA4sWLKS8vp6Wl5bDt5s+fT1xcqDt50qRJGIZBW1tbOIceMoemc02S6VxCCCEiKKwzbKfTSXZ2NiaTCQCTyURWVhZOp5P09KOfkb744osUFhaSk5PTp2M5HInhNPUImZlJX7iNYRiU72/lpAmZ5OVKYH9eb2oojk9qGD6pYXikfuEbqhqG3SXeFx988AH33nsvjz32WJ/f29zsQteNAWlHZmYSjY2dX7jdwRYPB5s9nDuroFfbx5Le1lAcm9QwfFLD8Ej9wjeQNVRV5bgnp2F1iefm5lJfX4+maQBomkZDQwO5ublHbLtlyxZ+9rOfsWrVKsaNGxfOYYeMTOcSQggxXIQV2A6Hg+LiYsrKygAoKyujuLj4iO7wrVu3cs0113DfffcxZcqUcA45pGQ6lxBCiOEi7HnYt9xyC0899RQLFy7kqaee4te//jUAy5cvZ9u2bQD8+te/pquri5tvvpklS5awZMkSdu3aFe6hB9Wh6Vxydi2EEGI4CPsadlFREc8999wRrz/yyCM9z1944YVwDzPkZDqXEEKI4URWOjsGmc4lhBBiOJHAPorD785linRzhBBCCAnso5G7cwkhhBhuJLCPQqZzCSGEGG4ksI9CpnMJIYQYbiSwP0emcwkhhBiOJLA/Z9OuBoKazrTxEthCCCGGDwnsz9B1g1fe309BZgLFo9Mi3RwhhBCihwT2Z2za3Yiz2cPiuWNQFSXSzRFCCCF6SGB30w2Dl9/dR056PLMnZUW6OUIIIcRhJLC7fbyniQONLhbPHY2qytm1EEKI4UUCm9DKZi+9t4+s1DhOPSE70s0RQgghjiCBDWyrbGH/wU4WnTYakyolEUIIMfzEfDoZhsHL71XhSLYx98ScSDdHCCGEOKqYD+yd+1upqO3gy3NGYzbFfDmEEEIMUzGfUC+/t4+URCvzp+VGuilCCCHEMcV0YO+uaWNndRtfPnW03EZTCCHEsBbTgf3ye/tIjrdw5vS8SDdFCCGEOK6YDezKug4+qWph4SmF2Cxydi2EEGJ4i9nALntvHwl2M2fNyI90U4QQQogvFJOBXVnbzkd7m1hw8ijibOZIN0cIIYT4QjEZ2H97YxdxNhPnziqIdFOEEEKIXom5wK5tdPHeVifnzBpFvN0S6eYIIYQQvRJzge1s9pCebOdLJ4+KdFOEEEKIXou5C7izJ2fxpbljaWlxR7opQgghRK/F3Bk2gEmWIBVCCBFlJLmEEEKIKCCBLYQQQkQBCWwhhBAiCkhgCyGEEFFAAlsIIYSIAlEzrUtVlWG9v1gkNQyf1DB8UsPwSP3CN1A1/KL9KIZhGANyJCGEEEIMGukSF0IIIaKABLYQQggRBSSwhRBCiCgggS2EEEJEAQlsIYQQIgpIYAshhBBRQAJbCCGEiAIS2EIIIUQUkMAWQgghokDMBXZVVRUXX3wxCxcu5OKLL2bfvn2RbtKwVlpaSklJCZMmTWL37t09r0sde6+1tZXly5ezcOFCLrjgAq688kpaWloAqWNvrVixggsvvJClS5eybNkyduzYAUj9+uOPf/zjYT/PUsPeKykp4bzzzmPJkiUsWbKEd955BxjCGhox5rLLLjNefPFFwzAM48UXXzQuu+yyCLdoeNu4caNRV1dnnH322cauXbt6Xpc69l5ra6uxfv36ns/vuOMO4xe/+IVhGFLH3uro6Oh5/vrrrxtLly41DEPq11fbt283Lr/8cuOss87q+XmWGvbe538PHjJUNYypM+zm5mbKy8tZvHgxAIsXL6a8vLznbEccafbs2eTm5h72mtSxb1JTUzn11FN7Pp8+fTp1dXVSxz5ISkrqee5yuVAURerXR36/n1tvvZWVK1eiKKGbTEgNwzeUNYyau3UNBKfTSXZ2NiaTCQCTyURWVhZOp5P09PQIty56SB37T9d1nnnmGUpKSqSOffTLX/6Sd999F8Mw+POf/yz166N7772XCy+8kFGjRvW8JjXsu+uuuw7DMJg1axbXXnvtkNYwps6whYi02267jfj4eC699NJINyXq/OY3v+Gtt97immuu4c4774x0c6LKli1b2LZtG8uWLYt0U6La008/zUsvvcQLL7yAYRjceuutQ3r8mArs3Nxc6uvr0TQNAE3TaGhoOKLLVxyf1LF/SktL2b9/P/fccw+qqkod+2np0qVs2LCBnJwcqV8vbdy4kcrKSs455xxKSko4ePAgl19+OdXV1VLDPjhUF6vVyrJly9i8efOQ/hzHVGA7HA6Ki4spKysDoKysjOLiYun66SOpY9/dfffdbN++nVWrVmG1WgGpY2+53W6cTmfP52vXriUlJUXq1wdXXHEF69atY+3ataxdu5acnBweffRRFi1aJDXsJY/HQ2dnJwCGYbBmzRqKi4uH9PtQMQzDGPC9DmMVFRXccMMNdHR0kJycTGlpKePGjYt0s4at22+/nddee42mpibS0tJITU3llVdekTr2wZ49e1i8eDFjxozBbrcDUFBQwKpVq6SOvdDU1MSKFSvwer2oqkpKSgrXX389U6ZMkfr1U0lJCQ899BATJ06UGvZSTU0NV111FZqmoes6RUVF/OpXvyIrK2vIahhzgS2EEEJEo5jqEhdCCCGilQS2EEIIEQUksIUQQogoIIEthBBCRAEJbCGEECIKSGALIYQQUUACWwghhIgCEthCCCFEFPj/KvEjWwdWYhAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt \n",
    "import numpy as np \n",
    "sns.set()\n",
    "\n",
    "f = plt.figure(figsize=(7, 7))\n",
    "gs = f.add_gridspec(2, 1)\n",
    "x = np.linspace(0, N_EPOCHS,N_EPOCHS)\n",
    "with sns.axes_style(\"darkgrid\"):\n",
    "    ax = f.add_subplot(gs[0])\n",
    "    plt.plot(x,t_l, label='Train Loss')\n",
    "    plt.plot(x,v_l, label='Valid Loss')\n",
    "    plt.title(\"Loss\")\n",
    "    plt.legend()\n",
    "        \n",
    "    ax = f.add_subplot(gs[1])\n",
    "    plt.plot(x, t_f1, label='Train F1 score')\n",
    "    plt.plot(x, v_f1, label='Valid F1 score')\n",
    "    plt.title(\"F1 score\")\n",
    "    plt.legend()\n",
    "\n",
    "       \n",
    "f.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-zuC2ENq7cYH"
   },
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "1zgluJGp7cYI",
    "outputId": "13ac5e3e-4995-43f8-bc61-b6417c98ce52"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss: 0.087 |  Test F1 score: 76.27%\n"
     ]
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('tut2-model_ner.pt'))\n",
    "f1, eval_loss, preds, labels= evaluate(model, test_loader, criterion)\n",
    "\n",
    "print(f'Test Loss: {eval_loss:.3f} |  Test F1 score: {f1*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "id": "7q4BAZIE-AKs",
    "outputId": "23ba6bfd-ff83-4eb7-868e-759b2dd3e1ca"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           precision    recall  f1-score   support\n",
      "\n",
      "      Cause-Effect(e1,e2)       0.90      0.90      0.90       134\n",
      "      Cause-Effect(e2,e1)       0.89      0.90      0.90       194\n",
      "   Component-Whole(e1,e2)       0.82      0.78      0.80       162\n",
      "   Component-Whole(e2,e1)       0.75      0.71      0.73       150\n",
      " Content-Container(e1,e2)       0.74      0.91      0.82       153\n",
      " Content-Container(e2,e1)       0.88      0.74      0.81        39\n",
      "Entity-Destination(e1,e2)       0.87      0.87      0.87       291\n",
      "Entity-Destination(e2,e1)       0.00      0.00      0.00         1\n",
      "     Entity-Origin(e1,e2)       0.78      0.89      0.83       211\n",
      "     Entity-Origin(e2,e1)       0.93      0.79      0.85        47\n",
      " Instrument-Agency(e1,e2)       1.00      0.05      0.09        22\n",
      " Instrument-Agency(e2,e1)       0.71      0.71      0.71       134\n",
      " Member-Collection(e1,e2)       0.00      0.00      0.00        32\n",
      " Member-Collection(e2,e1)       0.74      0.93      0.82       201\n",
      "     Message-Topic(e1,e2)       0.79      0.86      0.82       210\n",
      "     Message-Topic(e2,e1)       0.78      0.55      0.64        51\n",
      "                    Other       0.51      0.48      0.49       454\n",
      "  Product-Producer(e1,e2)       0.79      0.77      0.78       108\n",
      "  Product-Producer(e2,e1)       0.71      0.65      0.68       123\n",
      "\n",
      "                 accuracy                           0.75      2717\n",
      "                macro avg       0.71      0.66      0.66      2717\n",
      "             weighted avg       0.74      0.75      0.74      2717\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "predict =  preds\n",
    "true =  labels\n",
    "\n",
    "positive_labels = [i for i in range(len(rel2id)) if i != 0]\n",
    "positive_names = [list(rel2id.keys())[list(rel2id.values()).index(i)] for i in range(len(rel2id)) if i != 0]\n",
    "print(classification_report(\n",
    "                y_true=true,\n",
    "                y_pred=predict,\n",
    "                labels=positive_labels,\n",
    "                target_names=positive_names\n",
    "            ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "id": "EiqgErdJ-G4C",
    "outputId": "7ff30fd7-56a7-4901-fdb9-c75e81d98ad3"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cause-Effect(e1,e2)</th>\n",
       "      <th>Cause-Effect(e2,e1)</th>\n",
       "      <th>Component-Whole(e1,e2)</th>\n",
       "      <th>Component-Whole(e2,e1)</th>\n",
       "      <th>Content-Container(e1,e2)</th>\n",
       "      <th>Content-Container(e2,e1)</th>\n",
       "      <th>Entity-Destination(e1,e2)</th>\n",
       "      <th>Entity-Destination(e2,e1)</th>\n",
       "      <th>Entity-Origin(e1,e2)</th>\n",
       "      <th>Entity-Origin(e2,e1)</th>\n",
       "      <th>Instrument-Agency(e1,e2)</th>\n",
       "      <th>Instrument-Agency(e2,e1)</th>\n",
       "      <th>Member-Collection(e1,e2)</th>\n",
       "      <th>Member-Collection(e2,e1)</th>\n",
       "      <th>Message-Topic(e1,e2)</th>\n",
       "      <th>Message-Topic(e2,e1)</th>\n",
       "      <th>Other</th>\n",
       "      <th>Product-Producer(e1,e2)</th>\n",
       "      <th>Product-Producer(e2,e1)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Cause-Effect(e1,e2)</th>\n",
       "      <td>120</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cause-Effect(e2,e1)</th>\n",
       "      <td>1</td>\n",
       "      <td>175</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Component-Whole(e1,e2)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>126</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Component-Whole(e2,e1)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>106</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Content-Container(e1,e2)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>139</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Content-Container(e2,e1)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Entity-Destination(e1,e2)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>253</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Entity-Destination(e2,e1)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Entity-Origin(e1,e2)</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Entity-Origin(e2,e1)</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Instrument-Agency(e1,e2)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Instrument-Agency(e2,e1)</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Member-Collection(e1,e2)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Member-Collection(e2,e1)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>186</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Message-Topic(e1,e2)</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>181</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Message-Topic(e2,e1)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Other</th>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>41</td>\n",
       "      <td>27</td>\n",
       "      <td>4</td>\n",
       "      <td>218</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Product-Producer(e1,e2)</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>83</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Product-Producer(e2,e1)</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Cause-Effect(e1,e2)  Cause-Effect(e2,e1)  \\\n",
       "Cause-Effect(e1,e2)                        120                    3   \n",
       "Cause-Effect(e2,e1)                          1                  175   \n",
       "Component-Whole(e1,e2)                       0                    0   \n",
       "Component-Whole(e2,e1)                       0                    0   \n",
       "Content-Container(e1,e2)                     0                    0   \n",
       "Content-Container(e2,e1)                     0                    0   \n",
       "Entity-Destination(e1,e2)                    0                    0   \n",
       "Entity-Destination(e2,e1)                    0                    0   \n",
       "Entity-Origin(e1,e2)                         0                    4   \n",
       "Entity-Origin(e2,e1)                         1                    0   \n",
       "Instrument-Agency(e1,e2)                     0                    0   \n",
       "Instrument-Agency(e2,e1)                     1                    1   \n",
       "Member-Collection(e1,e2)                     0                    0   \n",
       "Member-Collection(e2,e1)                     0                    0   \n",
       "Message-Topic(e1,e2)                         2                    0   \n",
       "Message-Topic(e2,e1)                         0                    0   \n",
       "Other                                        8                   10   \n",
       "Product-Producer(e1,e2)                      0                    4   \n",
       "Product-Producer(e2,e1)                      1                    0   \n",
       "\n",
       "                           Component-Whole(e1,e2)  Component-Whole(e2,e1)  \\\n",
       "Cause-Effect(e1,e2)                             0                       2   \n",
       "Cause-Effect(e2,e1)                             0                       0   \n",
       "Component-Whole(e1,e2)                        126                       4   \n",
       "Component-Whole(e2,e1)                          5                     106   \n",
       "Content-Container(e1,e2)                        1                       1   \n",
       "Content-Container(e2,e1)                        0                       4   \n",
       "Entity-Destination(e1,e2)                       3                       0   \n",
       "Entity-Destination(e2,e1)                       0                       0   \n",
       "Entity-Origin(e1,e2)                            0                       0   \n",
       "Entity-Origin(e2,e1)                            0                       1   \n",
       "Instrument-Agency(e1,e2)                        0                       1   \n",
       "Instrument-Agency(e2,e1)                        0                       1   \n",
       "Member-Collection(e1,e2)                        2                       0   \n",
       "Member-Collection(e2,e1)                        0                       2   \n",
       "Message-Topic(e1,e2)                            0                       0   \n",
       "Message-Topic(e2,e1)                            1                       1   \n",
       "Other                                          11                      15   \n",
       "Product-Producer(e1,e2)                         2                       0   \n",
       "Product-Producer(e2,e1)                         2                       4   \n",
       "\n",
       "                           Content-Container(e1,e2)  Content-Container(e2,e1)  \\\n",
       "Cause-Effect(e1,e2)                               0                         0   \n",
       "Cause-Effect(e2,e1)                               0                         0   \n",
       "Component-Whole(e1,e2)                            5                         1   \n",
       "Component-Whole(e2,e1)                            0                         1   \n",
       "Content-Container(e1,e2)                        139                         0   \n",
       "Content-Container(e2,e1)                          0                        29   \n",
       "Entity-Destination(e1,e2)                        18                         0   \n",
       "Entity-Destination(e2,e1)                         0                         0   \n",
       "Entity-Origin(e1,e2)                              1                         0   \n",
       "Entity-Origin(e2,e1)                              0                         0   \n",
       "Instrument-Agency(e1,e2)                          0                         0   \n",
       "Instrument-Agency(e2,e1)                          0                         0   \n",
       "Member-Collection(e1,e2)                          0                         0   \n",
       "Member-Collection(e2,e1)                          0                         0   \n",
       "Message-Topic(e1,e2)                              0                         0   \n",
       "Message-Topic(e2,e1)                              0                         0   \n",
       "Other                                            24                         2   \n",
       "Product-Producer(e1,e2)                           0                         0   \n",
       "Product-Producer(e2,e1)                           1                         0   \n",
       "\n",
       "                           Entity-Destination(e1,e2)  \\\n",
       "Cause-Effect(e1,e2)                                0   \n",
       "Cause-Effect(e2,e1)                                0   \n",
       "Component-Whole(e1,e2)                             1   \n",
       "Component-Whole(e2,e1)                             2   \n",
       "Content-Container(e1,e2)                           4   \n",
       "Content-Container(e2,e1)                           0   \n",
       "Entity-Destination(e1,e2)                        253   \n",
       "Entity-Destination(e2,e1)                          1   \n",
       "Entity-Origin(e1,e2)                               4   \n",
       "Entity-Origin(e2,e1)                               0   \n",
       "Instrument-Agency(e1,e2)                           0   \n",
       "Instrument-Agency(e2,e1)                           2   \n",
       "Member-Collection(e1,e2)                           1   \n",
       "Member-Collection(e2,e1)                           0   \n",
       "Message-Topic(e1,e2)                               0   \n",
       "Message-Topic(e2,e1)                               0   \n",
       "Other                                             22   \n",
       "Product-Producer(e1,e2)                            0   \n",
       "Product-Producer(e2,e1)                            1   \n",
       "\n",
       "                           Entity-Destination(e2,e1)  Entity-Origin(e1,e2)  \\\n",
       "Cause-Effect(e1,e2)                                0                     0   \n",
       "Cause-Effect(e2,e1)                                0                     6   \n",
       "Component-Whole(e1,e2)                             0                     0   \n",
       "Component-Whole(e2,e1)                             0                     1   \n",
       "Content-Container(e1,e2)                           0                     2   \n",
       "Content-Container(e2,e1)                           0                     0   \n",
       "Entity-Destination(e1,e2)                          0                     0   \n",
       "Entity-Destination(e2,e1)                          0                     0   \n",
       "Entity-Origin(e1,e2)                               0                   187   \n",
       "Entity-Origin(e2,e1)                               0                     1   \n",
       "Instrument-Agency(e1,e2)                           0                     0   \n",
       "Instrument-Agency(e2,e1)                           0                     2   \n",
       "Member-Collection(e1,e2)                           0                     1   \n",
       "Member-Collection(e2,e1)                           0                     1   \n",
       "Message-Topic(e1,e2)                               0                     3   \n",
       "Message-Topic(e2,e1)                               0                     1   \n",
       "Other                                              0                    27   \n",
       "Product-Producer(e1,e2)                            0                     7   \n",
       "Product-Producer(e2,e1)                            0                     1   \n",
       "\n",
       "                           Entity-Origin(e2,e1)  Instrument-Agency(e1,e2)  \\\n",
       "Cause-Effect(e1,e2)                           0                         0   \n",
       "Cause-Effect(e2,e1)                           0                         0   \n",
       "Component-Whole(e1,e2)                        0                         0   \n",
       "Component-Whole(e2,e1)                        0                         0   \n",
       "Content-Container(e1,e2)                      0                         0   \n",
       "Content-Container(e2,e1)                      1                         0   \n",
       "Entity-Destination(e1,e2)                     0                         0   \n",
       "Entity-Destination(e2,e1)                     0                         0   \n",
       "Entity-Origin(e1,e2)                          0                         0   \n",
       "Entity-Origin(e2,e1)                         37                         0   \n",
       "Instrument-Agency(e1,e2)                      0                         1   \n",
       "Instrument-Agency(e2,e1)                      0                         0   \n",
       "Member-Collection(e1,e2)                      0                         0   \n",
       "Member-Collection(e2,e1)                      0                         0   \n",
       "Message-Topic(e1,e2)                          0                         0   \n",
       "Message-Topic(e2,e1)                          0                         0   \n",
       "Other                                         2                         0   \n",
       "Product-Producer(e1,e2)                       0                         0   \n",
       "Product-Producer(e2,e1)                       0                         0   \n",
       "\n",
       "                           Instrument-Agency(e2,e1)  Member-Collection(e1,e2)  \\\n",
       "Cause-Effect(e1,e2)                               0                         0   \n",
       "Cause-Effect(e2,e1)                               0                         0   \n",
       "Component-Whole(e1,e2)                            0                         0   \n",
       "Component-Whole(e2,e1)                            9                         0   \n",
       "Content-Container(e1,e2)                          0                         0   \n",
       "Content-Container(e2,e1)                          0                         0   \n",
       "Entity-Destination(e1,e2)                         1                         0   \n",
       "Entity-Destination(e2,e1)                         0                         0   \n",
       "Entity-Origin(e1,e2)                              0                         0   \n",
       "Entity-Origin(e2,e1)                              0                         0   \n",
       "Instrument-Agency(e1,e2)                          1                         0   \n",
       "Instrument-Agency(e2,e1)                         95                         0   \n",
       "Member-Collection(e1,e2)                          0                         0   \n",
       "Member-Collection(e2,e1)                          0                         0   \n",
       "Message-Topic(e1,e2)                              1                         0   \n",
       "Message-Topic(e2,e1)                              0                         0   \n",
       "Other                                            18                         0   \n",
       "Product-Producer(e1,e2)                           0                         0   \n",
       "Product-Producer(e2,e1)                           8                         0   \n",
       "\n",
       "                           Member-Collection(e2,e1)  Message-Topic(e1,e2)  \\\n",
       "Cause-Effect(e1,e2)                               0                     1   \n",
       "Cause-Effect(e2,e1)                               0                     0   \n",
       "Component-Whole(e1,e2)                            8                     1   \n",
       "Component-Whole(e2,e1)                            5                     3   \n",
       "Content-Container(e1,e2)                          0                     0   \n",
       "Content-Container(e2,e1)                          1                     1   \n",
       "Entity-Destination(e1,e2)                         0                     0   \n",
       "Entity-Destination(e2,e1)                         0                     0   \n",
       "Entity-Origin(e1,e2)                              0                     1   \n",
       "Entity-Origin(e2,e1)                              0                     0   \n",
       "Instrument-Agency(e1,e2)                          1                     0   \n",
       "Instrument-Agency(e2,e1)                          0                     1   \n",
       "Member-Collection(e1,e2)                          6                     0   \n",
       "Member-Collection(e2,e1)                        186                     1   \n",
       "Message-Topic(e1,e2)                              1                   181   \n",
       "Message-Topic(e2,e1)                              0                     9   \n",
       "Other                                            41                    27   \n",
       "Product-Producer(e1,e2)                           0                     2   \n",
       "Product-Producer(e2,e1)                           2                     2   \n",
       "\n",
       "                           Message-Topic(e2,e1)  Other  \\\n",
       "Cause-Effect(e1,e2)                           0      8   \n",
       "Cause-Effect(e2,e1)                           0      8   \n",
       "Component-Whole(e1,e2)                        0     16   \n",
       "Component-Whole(e2,e1)                        0     15   \n",
       "Content-Container(e1,e2)                      0      6   \n",
       "Content-Container(e2,e1)                      0      3   \n",
       "Entity-Destination(e1,e2)                     0     15   \n",
       "Entity-Destination(e2,e1)                     0      0   \n",
       "Entity-Origin(e1,e2)                          0     13   \n",
       "Entity-Origin(e2,e1)                          0      4   \n",
       "Instrument-Agency(e1,e2)                      0     15   \n",
       "Instrument-Agency(e2,e1)                      0     24   \n",
       "Member-Collection(e1,e2)                      2     19   \n",
       "Member-Collection(e2,e1)                      0     10   \n",
       "Message-Topic(e1,e2)                          1     21   \n",
       "Message-Topic(e2,e1)                         28     10   \n",
       "Other                                         4    218   \n",
       "Product-Producer(e1,e2)                       1      5   \n",
       "Product-Producer(e2,e1)                       0     21   \n",
       "\n",
       "                           Product-Producer(e1,e2)  Product-Producer(e2,e1)  \n",
       "Cause-Effect(e1,e2)                              0                        0  \n",
       "Cause-Effect(e2,e1)                              4                        0  \n",
       "Component-Whole(e1,e2)                           0                        0  \n",
       "Component-Whole(e2,e1)                           0                        3  \n",
       "Content-Container(e1,e2)                         0                        0  \n",
       "Content-Container(e2,e1)                         0                        0  \n",
       "Entity-Destination(e1,e2)                        0                        1  \n",
       "Entity-Destination(e2,e1)                        0                        0  \n",
       "Entity-Origin(e1,e2)                             1                        0  \n",
       "Entity-Origin(e2,e1)                             0                        3  \n",
       "Instrument-Agency(e1,e2)                         3                        0  \n",
       "Instrument-Agency(e2,e1)                         0                        7  \n",
       "Member-Collection(e1,e2)                         0                        1  \n",
       "Member-Collection(e2,e1)                         0                        1  \n",
       "Message-Topic(e1,e2)                             0                        0  \n",
       "Message-Topic(e2,e1)                             1                        0  \n",
       "Other                                           13                       12  \n",
       "Product-Producer(e1,e2)                         83                        4  \n",
       "Product-Producer(e2,e1)                          0                       80  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "confusion = confusion_matrix(true, predict)\n",
    "l = positive_names\n",
    "\n",
    "confusion_df =pd.DataFrame(confusion)\n",
    "\n",
    "confusion_df.columns=l\n",
    "s = pd.Series(l)\n",
    "confusion_df = confusion_df.set_index([s])\n",
    "\n",
    "\n",
    "confusion_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aye1d2Md7cYS"
   },
   "source": [
    "## Références : \n",
    "\n",
    " - [Attention-Based Bidirectional Long Short-Term Memory Networks for\n",
    "Relation Classification](https://www.aclweb.org/anthology/P16-2034.pdf)\n",
    " - [Att-BLSTM-relation-extraction](https://github.com/onehaitao/Att-BLSTM-relation-extraction)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "2-BiLSTM_Att-RE.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}