research_project.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "id": "8d2eb7f2",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8d2eb7f2",
        "outputId": "47297171-2510-4da2-e865-8cbb8f1e7cf3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.20.0-py3-none-any.whl (4.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.4 MB 6.9 MB/s \n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 10.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.1)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 38.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.7.0-py3-none-any.whl (86 kB)\n",
            "\u001b[K     |████████████████████████████████| 86 kB 3.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.6.15)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Installing collected packages: pyyaml, tokenizers, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.7.0 pyyaml-6.0 tokenizers-0.12.1 transformers-4.20.0\n",
            "Tensorflow version 2.8.2\n",
            "Running on TPU  ['10.122.9.186:8470']\n",
            "INFO:tensorflow:Deallocate tpu buffers before initializing tpu system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Deallocate tpu buffers before initializing tpu system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.122.9.186:8470\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.122.9.186:8470\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "import tensorflow as tf\n",
        "print(\"Tensorflow version \" + tf.__version__)\n",
        "\n",
        "try:\n",
        "  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n",
        "  print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\n",
        "except ValueError:\n",
        "  raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n",
        "\n",
        "tf.config.experimental_connect_to_cluster(tpu)\n",
        "tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
        "from os import truncate\n",
        "from transformers import BertTokenizer\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from transformers import TFBertForSequenceClassification\n",
        "from matplotlib import pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "7196fcb7",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 383,
          "referenced_widgets": [
            "8d7f045edeb649d7941ac905124e6d5c",
            "ce24ef43014b4f53ac6752c282b04f89",
            "aa783f4a0df9401c8000f8ac1461a012",
            "f905c1faebe04940994f248f5197a8f3",
            "4fa5abe45d4c43049f681fe4c3769e5f",
            "3176fb85dd86418abd8a94eeb26a5f8b",
            "385324a74cf344a59631878a5081f10f",
            "a9db5dedd526493796f04be709dc00f4",
            "4217aef3643949baa87d820462a8b851",
            "cd098d2dae0d4a49b47fc2ea683b1b18",
            "47fb20e2ec2a42ac9742c4f31336dfb8",
            "720ea5e89dfe4697ac1577da48a1cfaa",
            "94a7674878b84f418d3476c62243a63e",
            "135f699e866c4a2582ecff75a99bce12",
            "1c7fb8eadfd54281aa70650de29df46d",
            "f92a53f977bc48819a7d2a4b9467ba98",
            "52efd69e07414618816fa8d5570b3d80",
            "3352c4ac50124a9db8819b5ec5c8c753",
            "757ed4f431714205805b8967ca6a2ada",
            "da66a1fa86364ae9807aebf283ac569b",
            "129be69475b34e4f812cf5dd5343d670",
            "a96e6ffd181d461aa29d6b1c07a04fdd",
            "16abfbb1cea6404eb3b5c42d11eb8159",
            "00a9d22d0c1c4f1aaba87a0857198d5b",
            "d3cca8fe0b604587ac69747cfb92edf9",
            "20acc34a6a664493954fe875f1a0b40d",
            "fa02013080794258a79965fb5f222bb3",
            "9b7cc371059d4272bd46aaede592ca9e",
            "a5d3ee11f4fe45de99e77878643d2b25",
            "2e8bbfa35f754a4d8b002892d020f504",
            "89c2fe0958ed4067b72ea32036c17137",
            "4b9c1a9a1ae845f2a988a4e2c9e3de30",
            "fb0b0d6aae374ce8a1b4dab0803e9fea"
          ]
        },
        "id": "7196fcb7",
        "outputId": "3fe8b587-b986-41f2-b2b0-a09aaefd8787"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8d7f045edeb649d7941ac905124e6d5c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "720ea5e89dfe4697ac1577da48a1cfaa"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "16abfbb1cea6404eb3b5c42d11eb8159"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Encoding processing\n",
            "Encoding processing\n",
            "Encoding processing\n",
            "Encoding processing\n",
            "Encoding processing\n",
            "Encoding processing\n",
            "Encoding processing\n",
            "Encoding processing\n",
            "Encoding processing\n",
            "Encoding processing\n",
            "train_inp1 length: (446, 20), train_mask length: (446, 20)\n",
            "[CLS] how old is a child movie ticket [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]\n",
            "train_label1 length: (446, 1)\n",
            "val_inp1 length: (112, 20), val_label1 length: (112, 1)\n",
            "test_inp length: (62, 20), test_label length: (62, 1)\n"
          ]
        }
      ],
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "from sklearn.model_selection import KFold\n",
        "\n",
        "# Set the split ratio to 9:1 for training and test set\n",
        "def split_test_set(df):\n",
        "  x, train_set = train_test_split(df, \n",
        "        stratify=df['label'],\n",
        "        test_size=0.1, \n",
        "        random_state=42)\n",
        "  return x, train_set\n",
        "\n",
        "# Read file\n",
        "df_raw = pd.read_excel(\"classifier1.xlsx\",names=[\"query\",\"intent\",\"label\"])\n",
        "# df_raw = pd.read_excel(\"classifier2.xlsx\",names=[\"query\",\"intent\",\"label\"])\n",
        "# df_raw = pd.read_excel(\"classifier3.xlsx\",names=[\"query\",\"intent\",\"label\"])\n",
        "# label\n",
        "\n",
        "# Add another column y in the dataframe for the numeric labels \n",
        "# Classifier 1\n",
        "df_label = pd.DataFrame({\"label\":[\"G\",\"D\"],\"y\":list(range(2))})\n",
        "# Classifier 2\n",
        "# df_label = pd.DataFrame({\"label\":[\"PG\",\"G\"],\"y\":list(range(2))})\n",
        "# Classifier 3\n",
        "# df_label = pd.DataFrame({\"label\":[\"PD\",\"ND\",\"D\"],\"y\":list(range(3))})\n",
        "df_raw = pd.merge(df_raw,df_label,on=\"label\",how=\"left\")\n",
        "\n",
        "# Randomly select a subset of data from each user intent\n",
        "size = 20       # sample size\n",
        "replace = True  # with replacement\n",
        "fn = lambda obj: obj.loc[np.random.choice(obj.index, size, replace),:]\n",
        "df_raw = df_raw.groupby('intent', as_index=False).apply(fn)\n",
        "\n",
        "# Split the training set and the test set\n",
        "x, test_set = split_test_set(df_raw)\n",
        "# Apply 5-fold cross validation for the training set\n",
        "kf = KFold(n_splits=5, random_state=42, shuffle=True)\n",
        "\n",
        "# Get training sets and validation sets\n",
        "split_list = list(kf.split(x))\n",
        "train_1 = df_raw.iloc[split_list[0][0]]\n",
        "validation_1 = df_raw.iloc[split_list[0][1]]\n",
        "train_2 = df_raw.iloc[split_list[1][0]]\n",
        "validation_2 = df_raw.iloc[split_list[1][1]]\n",
        "train_3 = df_raw.iloc[split_list[2][0]]\n",
        "validation_3 = df_raw.iloc[split_list[2][1]]\n",
        "train_4 = df_raw.iloc[split_list[3][0]]\n",
        "validation_4 = df_raw.iloc[split_list[3][1]]\n",
        "train_5 = df_raw.iloc[split_list[4][0]]\n",
        "validation_5 = df_raw.iloc[split_list[4][1]]\n",
        "\n",
        "\n",
        "# Encode the texts, samples include user intents\n",
        "def encode_with(ds, limit=-1):\n",
        "    input_ids_list = []\n",
        "    token_type_ids_list = []\n",
        "    attention_mask_list = []\n",
        "    label_list = []\n",
        "    if (limit > 0):\n",
        "        ds = ds.take(limit)\n",
        "    for index, row in ds.iterrows():\n",
        "        query = row[\"query\"]\n",
        "        label = row[\"y\"]\n",
        "        intent = row[\"intent\"]\n",
        "        bert_input = tokenizer.encode_plus(query,intent,add_special_tokens=True,max_length=20,\n",
        "                                 padding='max_length',return_attention_mask=True,truncation='only_first')\n",
        "         \n",
        "        \n",
        "        bert_input = tokenizer.encode_plus(query,add_special_tokens=True,max_length=20,padding='max_length',return_attention_mask=True,truncation=True) \n",
        "  \n",
        "        \n",
        "        input_ids_list.append(bert_input['input_ids'])\n",
        "        token_type_ids_list.append(bert_input['token_type_ids'])\n",
        "        attention_mask_list.append(bert_input['attention_mask'])\n",
        "        label_list.append([label])\n",
        "\n",
        "    input_ids=np.array(input_ids_list)\n",
        "    attention_masks=np.array(attention_mask_list)\n",
        "    labels=np.array(label_list)\n",
        "    token_types=np.array(token_type_ids_list)\n",
        "      \n",
        "    return input_ids,attention_masks,labels,token_types\n",
        "\n",
        "# Encode the texts, for classifiers that only use queries\n",
        "def encode_without(ds, limit=-1):\n",
        "    input_ids_list = []\n",
        "    token_type_ids_list = []\n",
        "    attention_mask_list = []\n",
        "    label_list = []\n",
        "    if (limit > 0):\n",
        "        ds = ds.take(limit)\n",
        "    for index, row in ds.iterrows():\n",
        "        query = row[\"query\"]\n",
        "        label = row[\"y\"]\n",
        "        \n",
        "        bert_input = tokenizer.encode_plus(query,add_special_tokens=True,max_length=20,padding='max_length',return_attention_mask=True,truncation=True) \n",
        "  \n",
        "        \n",
        "        input_ids_list.append(bert_input['input_ids'])\n",
        "        token_type_ids_list.append(bert_input['token_type_ids'])\n",
        "        attention_mask_list.append(bert_input['attention_mask'])\n",
        "        label_list.append([label])\n",
        "\n",
        "    input_ids=np.array(input_ids_list)\n",
        "    attention_masks=np.array(attention_mask_list)\n",
        "    labels=np.array(label_list)\n",
        "    token_types=np.array(token_type_ids_list)\n",
        "      \n",
        "    return input_ids,attention_masks,labels,token_types\n",
        "\n",
        "# Encode training set and validation set\n",
        "def encode_process(trainset, validationset, without):\n",
        "  print(\"Encoding processing\")\n",
        "  if without == False:\n",
        "    train_inp,train_mask,train_label,train_type = encode_with(trainset)\n",
        "    val_inp,val_mask,val_label,val_type=encode_with(validationset)\n",
        "  else:\n",
        "    train_inp,train_mask,train_label,train_type = encode_without(trainset)\n",
        "    val_inp,val_mask,val_label,val_type=encode_without(validationset)\n",
        "  return train_inp, train_mask, train_label, train_type, val_inp, val_mask, val_label, val_type\n",
        "\n",
        "# encode training and validation sets in each fold, with user intents\n",
        "train_inp1,train_mask1,train_label1,train_type1,val_inp1,val_mask1,val_label1,val_type1=encode_process(train_1,validation_1,False)\n",
        "train_inp2,train_mask2,train_label2,train_type2,val_inp2,val_mask2,val_label2,val_type2=encode_process(train_2,validation_2,False)\n",
        "train_inp3,train_mask3,train_label3,train_type3,val_inp3,val_mask3,val_label3,val_type3=encode_process(train_3,validation_3,False)\n",
        "train_inp4,train_mask4,train_label4,train_type4,val_inp4,val_mask4,val_label4,val_type4=encode_process(train_4,validation_4,False)\n",
        "train_inp5,train_mask5,train_label5,train_type5,val_inp5,val_mask5,val_label5,val_type5=encode_process(train_5,validation_5,False)\n",
        "\n",
        "# encode training and validation sets in each fold, without user intents\n",
        "train_inp1_wo,train_mask1_wo,train_label1_wo,train_type1_wo,val_inp1_wo,val_mask1_wo,val_label1_wo,val_type1_wo=encode_process(train_1,validation_1,True)\n",
        "train_inp2_wo,train_mask2_wo,train_label2_wo,train_type2_wo,val_inp2_wo,val_mask2_wo,val_label2_wo,val_type2_wo=encode_process(train_2,validation_2,True)\n",
        "train_inp3_wo,train_mask3_wo,train_label3_wo,train_type3_wo,val_inp3_wo,val_mask3_wo,val_label3_wo,val_type3_wo=encode_process(train_3,validation_3,True)\n",
        "train_inp4_wo,train_mask4_wo,train_label4_wo,train_type4_wo,val_inp4_wo,val_mask4_wo,val_label4_wo,val_type4_wo=encode_process(train_4,validation_4,True)\n",
        "train_inp5_wo,train_mask5_wo,train_label5_wo,train_type5_wo,val_inp5_wo,val_mask5_wo,val_label5_wo,val_type5_wo=encode_process(train_5,validation_5,True)\n",
        "\n",
        "test_inp, test_mask, test_label, test_type = encode_with(test_set)\n",
        "test_inp_wo, test_mask_wo, test_label_wo, test_type_wo = encode_without(test_set)\n",
        "\n",
        "print(\"train_inp1 length: {}, train_mask length: {}\".format(train_inp1.shape,train_mask1.shape))\n",
        "print(tokenizer.decode(train_inp1[1]))\n",
        "print(\"train_label1 length: {}\".format(train_label1.shape))\n",
        "print(\"val_inp1 length: {}, val_label1 length: {}\".format(val_inp1.shape,val_label1.shape))\n",
        "print(\"test_inp length: {}, test_label length: {}\".format(test_inp.shape,test_label.shape))\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "id": "2f412d6f",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2f412d6f",
        "outputId": "6de21f89-8f38-4fa8-bac2-d3f10fa3572a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "model with summary\n",
            "Model: \"tf_bert_for_sequence_classification_6\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " bert (TFBertMainLayer)      multiple                  109482240 \n",
            "                                                                 \n",
            " dropout_265 (Dropout)       multiple                  0         \n",
            "                                                                 \n",
            " classifier (Dense)          multiple                  1538      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 109,483,778\n",
            "Trainable params: 109,483,778\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n",
            "model without summary\n",
            "Model: \"tf_bert_for_sequence_classification_7\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " bert (TFBertMainLayer)      multiple                  109482240 \n",
            "                                                                 \n",
            " dropout_303 (Dropout)       multiple                  0         \n",
            "                                                                 \n",
            " classifier (Dense)          multiple                  1538      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 109,483,778\n",
            "Trainable params: 109,483,778\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n",
            "Training process, with usesr intents\n",
            "14/14 [==============================] - 102s 1s/step - loss: 0.3398 - accuracy: 0.9081 - val_loss: 0.1254 - val_accuracy: 0.9732\n",
            "14/14 [==============================] - 2s 124ms/step - loss: 0.1498 - accuracy: 0.9641 - val_loss: 0.1246 - val_accuracy: 0.9643\n",
            "14/14 [==============================] - 2s 124ms/step - loss: 0.1044 - accuracy: 0.9641 - val_loss: 0.0669 - val_accuracy: 0.9643\n",
            "14/14 [==============================] - 2s 126ms/step - loss: 0.0524 - accuracy: 0.9620 - val_loss: 0.0253 - val_accuracy: 0.9820\n",
            "14/14 [==============================] - 2s 125ms/step - loss: 0.0242 - accuracy: 0.9955 - val_loss: 0.0208 - val_accuracy: 1.0000\n",
            "Training process, without user intents\n",
            "14/14 [==============================] - 102s 1s/step - loss: 0.4679 - accuracy: 0.8767 - val_loss: 0.2381 - val_accuracy: 0.9732\n",
            "14/14 [==============================] - 2s 124ms/step - loss: 0.1898 - accuracy: 0.9641 - val_loss: 0.1264 - val_accuracy: 0.9643\n",
            "14/14 [==============================] - 2s 117ms/step - loss: 0.0928 - accuracy: 0.9731 - val_loss: 0.0639 - val_accuracy: 0.9911\n",
            "14/14 [==============================] - 2s 119ms/step - loss: 0.0419 - accuracy: 0.9933 - val_loss: 0.0210 - val_accuracy: 1.0000\n",
            "14/14 [==============================] - 2s 119ms/step - loss: 0.0278 - accuracy: 0.9978 - val_loss: 0.0191 - val_accuracy: 1.0000\n",
            "Test set evaluation, with user intents\n",
            "2/2 [==============================] - 1s 34ms/step - loss: 0.1031 - accuracy: 0.9839\n",
            "Test set evaluation, without user intents\n",
            "2/2 [==============================] - 1s 28ms/step - loss: 0.0399 - accuracy: 0.9839\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.03989721089601517, 0.9838709235191345]"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "# Set the hyperparameters\n",
        "learning_rate = 3e-5\n",
        "number_of_epochs = 1\n",
        "batch_size = 32\n",
        "with tpu_strategy.scope():\n",
        "\n",
        "# Model initialization with user intents\n",
        "  model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
        "  # model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=3)\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
        "  loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "  metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
        "  # model.layers[1].trainable = False\n",
        "\n",
        "  model.compile(optimizer=optimizer, loss=loss, metrics=[metric])\n",
        "\n",
        "  # Model initialization without user intents\n",
        "  model_wo = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
        "  # model_wo = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=3)\n",
        "\n",
        "  optimizer_wo = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
        "  loss_wo = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "  metric_wo = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
        "  # model_wo.layers[1].trainable = False\n",
        "  model_wo.compile(optimizer=optimizer_wo, loss=loss_wo, metrics=[metric_wo])\n",
        "\n",
        "\n",
        "print(\"model with summary\")\n",
        "print(model.summary())\n",
        "\n",
        "print(\"model without summary\")\n",
        "print(model_wo.summary())\n",
        "\n",
        "# Fit model\n",
        "print(\"Training process, with usesr intents\")\n",
        "bert_history = model.fit([train_inp1,train_mask1,train_type1],train_label1,batch_size=batch_size,epochs=number_of_epochs,validation_data=([val_inp1,val_mask1,val_type1],val_label1))\n",
        "bert_history = model.fit([train_inp2,train_mask2,train_type2],train_label2,batch_size=batch_size,epochs=number_of_epochs,validation_data=([val_inp2,val_mask2,val_type2],val_label2))\n",
        "bert_history = model.fit([train_inp3,train_mask3,train_type3],train_label3,batch_size=batch_size,epochs=number_of_epochs,validation_data=([val_inp3,val_mask3,val_type3],val_label3))\n",
        "bert_history = model.fit([train_inp4,train_mask4,train_type4],train_label4,batch_size=batch_size,epochs=number_of_epochs,validation_data=([val_inp4,val_mask4,val_type4],val_label4))\n",
        "bert_history = model.fit([train_inp5,train_mask5,train_type5],train_label5,batch_size=batch_size,epochs=number_of_epochs,validation_data=([val_inp5,val_mask5,val_type5],val_label5))\n",
        "\n",
        "print(\"Training process, without user intents\")\n",
        "bert_history = model_wo.fit([train_inp1_wo,train_mask1_wo,train_type1_wo],train_label1_wo,batch_size=batch_size,epochs=number_of_epochs,validation_data=([val_inp1_wo,val_mask1_wo,val_type1_wo],val_label1_wo))\n",
        "bert_history = model_wo.fit([train_inp2_wo,train_mask2_wo,train_type2_wo],train_label2_wo,batch_size=batch_size,epochs=number_of_epochs,validation_data=([val_inp2_wo,val_mask2_wo,val_type2_wo],val_label2_wo))\n",
        "bert_history = model_wo.fit([train_inp3_wo,train_mask3_wo,train_type3_wo],train_label3_wo,batch_size=batch_size,epochs=number_of_epochs,validation_data=([val_inp3_wo,val_mask3_wo,val_type3_wo],val_label3_wo))\n",
        "bert_history = model_wo.fit([train_inp4_wo,train_mask4_wo,train_type4_wo],train_label4_wo,batch_size=batch_size,epochs=number_of_epochs,validation_data=([val_inp4_wo,val_mask4_wo,val_type4_wo],val_label4_wo))\n",
        "bert_history = model_wo.fit([train_inp5_wo,train_mask5_wo,train_type5_wo],train_label5_wo,batch_size=batch_size,epochs=number_of_epochs,validation_data=([val_inp5_wo,val_mask5_wo,val_type5_wo],val_label5_wo))\n",
        "\n",
        "\n",
        "print('Test set evaluation, with user intents')\n",
        "model.evaluate([test_inp,test_mask,test_type],test_label)\n",
        "print('Test set evaluation, without user intents')\n",
        "model_wo.evaluate([test_inp_wo,test_mask_wo,test_type_wo],test_label_wo)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "wO358wHDyJo4",
      "metadata": {
        "id": "wO358wHDyJo4"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "collapsed_sections": [],
      "name": "rp.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8d7f045edeb649d7941ac905124e6d5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ce24ef43014b4f53ac6752c282b04f89",
              "IPY_MODEL_aa783f4a0df9401c8000f8ac1461a012",
              "IPY_MODEL_f905c1faebe04940994f248f5197a8f3"
            ],
            "layout": "IPY_MODEL_4fa5abe45d4c43049f681fe4c3769e5f"
          }
        },
        "ce24ef43014b4f53ac6752c282b04f89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3176fb85dd86418abd8a94eeb26a5f8b",
            "placeholder": "​",
            "style": "IPY_MODEL_385324a74cf344a59631878a5081f10f",
            "value": "Downloading: 100%"
          }
        },
        "aa783f4a0df9401c8000f8ac1461a012": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a9db5dedd526493796f04be709dc00f4",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4217aef3643949baa87d820462a8b851",
            "value": 231508
          }
        },
        "f905c1faebe04940994f248f5197a8f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd098d2dae0d4a49b47fc2ea683b1b18",
            "placeholder": "​",
            "style": "IPY_MODEL_47fb20e2ec2a42ac9742c4f31336dfb8",
            "value": " 226k/226k [00:00&lt;00:00, 1.07MB/s]"
          }
        },
        "4fa5abe45d4c43049f681fe4c3769e5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3176fb85dd86418abd8a94eeb26a5f8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "385324a74cf344a59631878a5081f10f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a9db5dedd526493796f04be709dc00f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4217aef3643949baa87d820462a8b851": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cd098d2dae0d4a49b47fc2ea683b1b18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "47fb20e2ec2a42ac9742c4f31336dfb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "720ea5e89dfe4697ac1577da48a1cfaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_94a7674878b84f418d3476c62243a63e",
              "IPY_MODEL_135f699e866c4a2582ecff75a99bce12",
              "IPY_MODEL_1c7fb8eadfd54281aa70650de29df46d"
            ],
            "layout": "IPY_MODEL_f92a53f977bc48819a7d2a4b9467ba98"
          }
        },
        "94a7674878b84f418d3476c62243a63e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_52efd69e07414618816fa8d5570b3d80",
            "placeholder": "​",
            "style": "IPY_MODEL_3352c4ac50124a9db8819b5ec5c8c753",
            "value": "Downloading: 100%"
          }
        },
        "135f699e866c4a2582ecff75a99bce12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_757ed4f431714205805b8967ca6a2ada",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_da66a1fa86364ae9807aebf283ac569b",
            "value": 28
          }
        },
        "1c7fb8eadfd54281aa70650de29df46d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_129be69475b34e4f812cf5dd5343d670",
            "placeholder": "​",
            "style": "IPY_MODEL_a96e6ffd181d461aa29d6b1c07a04fdd",
            "value": " 28.0/28.0 [00:00&lt;00:00, 671B/s]"
          }
        },
        "f92a53f977bc48819a7d2a4b9467ba98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "52efd69e07414618816fa8d5570b3d80": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3352c4ac50124a9db8819b5ec5c8c753": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "757ed4f431714205805b8967ca6a2ada": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da66a1fa86364ae9807aebf283ac569b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "129be69475b34e4f812cf5dd5343d670": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a96e6ffd181d461aa29d6b1c07a04fdd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "16abfbb1cea6404eb3b5c42d11eb8159": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_00a9d22d0c1c4f1aaba87a0857198d5b",
              "IPY_MODEL_d3cca8fe0b604587ac69747cfb92edf9",
              "IPY_MODEL_20acc34a6a664493954fe875f1a0b40d"
            ],
            "layout": "IPY_MODEL_fa02013080794258a79965fb5f222bb3"
          }
        },
        "00a9d22d0c1c4f1aaba87a0857198d5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9b7cc371059d4272bd46aaede592ca9e",
            "placeholder": "​",
            "style": "IPY_MODEL_a5d3ee11f4fe45de99e77878643d2b25",
            "value": "Downloading: 100%"
          }
        },
        "d3cca8fe0b604587ac69747cfb92edf9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e8bbfa35f754a4d8b002892d020f504",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_89c2fe0958ed4067b72ea32036c17137",
            "value": 570
          }
        },
        "20acc34a6a664493954fe875f1a0b40d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4b9c1a9a1ae845f2a988a4e2c9e3de30",
            "placeholder": "​",
            "style": "IPY_MODEL_fb0b0d6aae374ce8a1b4dab0803e9fea",
            "value": " 570/570 [00:00&lt;00:00, 8.88kB/s]"
          }
        },
        "fa02013080794258a79965fb5f222bb3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b7cc371059d4272bd46aaede592ca9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a5d3ee11f4fe45de99e77878643d2b25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2e8bbfa35f754a4d8b002892d020f504": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "89c2fe0958ed4067b72ea32036c17137": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4b9c1a9a1ae845f2a988a4e2c9e3de30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fb0b0d6aae374ce8a1b4dab0803e9fea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    },
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 5
}