Text-Sentiment-Analysis-using-BERT.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/LuluW8071/Text-Sentiment-Analysis/blob/main/Text-Sentiment-Analysis-using-BERT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fMxocdwKAhgv"
      },
      "source": [
        "# Text-Sentiment-Analysis-using-BERT"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VFlkWVYFy-QA",
        "outputId": "bc11f971-81ed-4e5d-a2fd-5224ce686fe9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers[torch] in /usr/local/lib/python3.10/dist-packages (4.41.2)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.20.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (3.15.4)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.23.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2024.5.15)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.32.3)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.19.1)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.4.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (4.66.4)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.3.0+cu121)\n",
            "Requirement already satisfied: accelerate>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.32.1)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (16.1.0)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.0.3)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.16)\n",
            "Requirement already satisfied: fsspec[http]<=2024.5.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.5)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->transformers[torch]) (5.9.5)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.0->transformers[torch]) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2024.6.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (1.12.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (3.1.4)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (2.20.5)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (12.1.105)\n",
            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch->transformers[torch]) (2.3.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->transformers[torch]) (12.5.82)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->transformers[torch]) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->transformers[torch]) (1.3.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers[torch] datasets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "39k-BncjiZtY"
      },
      "source": [
        "## 1. Download and Load the dataset\n",
        "\n",
        "The dataset that the following script will download is a combination of the Yelp Polarity Dataset and the IMDb Movie Dataset. The Yelp Polarity Dataset has been preprocessed by selecting specific columns to create a dataset suitable for sentiment analysis. This preprocessed dataset has been merged with the IMDb Movie Dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n9vIPvXLiWNx",
        "outputId": "1bb910d9-0a80-4eb5-ef98-49d7feae9abb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From (original): https://drive.google.com/uc?id=1Jp3D5gdxGrwa5dHbr4p-pECrD8wi7vik\n",
            "From (redirected): https://drive.google.com/uc?id=1Jp3D5gdxGrwa5dHbr4p-pECrD8wi7vik&confirm=t&uuid=9e9d5b81-a1e6-40dd-8234-de5e75615211\n",
            "To: /content/sentiment_dataset.zip\n",
            "100%|██████████| 182M/182M [00:05<00:00, 31.4MB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Files extracted successfully to: ./dataset\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import gdown\n",
        "import zipfile\n",
        "\n",
        "file_url = 'https://drive.google.com/uc?id=1Jp3D5gdxGrwa5dHbr4p-pECrD8wi7vik'\n",
        "file_name = 'sentiment_dataset.zip'\n",
        "\n",
        "# Download the file from Google Drive\n",
        "gdown.download(file_url, file_name, quiet=False)\n",
        "extract_dir = './dataset'\n",
        "\n",
        "# Extract the zip file\n",
        "with zipfile.ZipFile(file_name, 'r') as zip_ref:\n",
        "    zip_ref.extractall(extract_dir)\n",
        "\n",
        "# Remove the zip file after extraction\n",
        "os.remove(file_name)\n",
        "print(\"Files extracted successfully to:\", extract_dir)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z_CiiyyauW5U"
      },
      "source": [
        ">__Note:__</br>\n",
        "**BERT** (Bidirectional Encoder Representations from Transformers) can indeed be trained on a relatively small dataset to yield improved results for certain tasks, especially when fine-tuning a pre-trained model, due to its powerful architecture. It is already pre-trained on larger datasets, possesses powerful contextual understanding, and benefits from effective regularization techniques such as dropout and attention mechanisms, which help prevent overfitting.\n",
        "\n",
        ">So, we can just take just `20000` datasets and train the **BERT** Model on it for our purpose."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "twsVt1DPihlH",
        "outputId": "ac7f0b63-752c-4765-a2c0-a50a7737b8c5"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(                                              review sentiment\n",
              " 0  The minicupcakes are cute and taste sweet but ...  negative\n",
              " 1  We had went to the old Satified Frog frequentl...  negative\n",
              " 2  Very unprofessional front of the house staff. ...  negative\n",
              " 3  We stopped by to check out the happy hour spec...  positive\n",
              " 4  The pear dessert crepe was super delicious!  T...  positive,\n",
              " (20000, 2))"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "import random\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "# Reduce to 10000 samples if you want your model to train faster (while loss may increase)\n",
        "samples = 20000\n",
        "\n",
        "# Read dataset and take random 20000 samples\n",
        "df = pd.read_csv(\"dataset/sentiment_combined.csv\")\n",
        "df = df.sample(n=samples, random_state=random.randint(0, 100))\n",
        "\n",
        "# Reset the index\n",
        "df.reset_index(drop=True, inplace=True)\n",
        "df.head(), df.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "id": "xHyN1x6Pih5u",
        "outputId": "57dc6965-42aa-4424-ae4b-6fa49528486a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"The minicupcakes are cute and taste sweet but the vanilla has a strange aftertaste from the frosting. The black and white is good though. Meanwhile their ice creams (cap'n ' crunch, salted caramel, cherry vanilla) are all atrocious. I've literally never tasted ice cream this bad. The salted caramel just tastes like salt, the cap n' crunch is crappy soggy peanut butter mush and the cherry vanilla just tastes like a bad imitation of Ben and Jerry's. The worst by far is not using the berry cap n crunch. Holstein's and Momofuku is much better.\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "df['review'][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SsFTo9aWi62B",
        "outputId": "865bbc98-0d63-40ce-f589-3d70fcbcd36b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "sentiment\n",
              "positive    10165\n",
              "negative     9835\n",
              "Name: count, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "df['sentiment'].value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bnkgCW7L1w_h"
      },
      "source": [
        "## 2. Text Pre-Processing\n",
        "\n",
        "- Cleaning up the text data by removing punctuation, extra spaces, and numbers.\n",
        "- Transform sentences into individual words, remove common words (known as \"stop words\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RQ6ShoeAlpPv",
        "outputId": "fdaa6521-3f5e-47b9-a266-9603f2e69415"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import re\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "nltk.download('stopwords')\n",
        "from collections import Counter\n",
        "\n",
        "# Precompile regular expressions for faster pre processing\n",
        "non_word_chars_pattern = re.compile(r\"[^\\w\\s]\")\n",
        "whitespace_pattern = re.compile(r\"\\s+\")\n",
        "digits_pattern = re.compile(r\"\\d\")\n",
        "username_pattern = re.compile(r\"@([^\\s]+)\")\n",
        "hashtags_pattern = re.compile(r\"#\\d+\")\n",
        "br_pattern = re.compile(r'<br\\s*/?>\\s*<br\\s*/?>')\n",
        "\n",
        "def preprocess_string(s):\n",
        "    # Remove all non-word characters (everything except numbers and letters)\n",
        "    s = non_word_chars_pattern.sub('', s)\n",
        "    # Replace all runs of whitespaces with single space\n",
        "    s = whitespace_pattern.sub(' ', s)\n",
        "    # Replace digits with no space\n",
        "    s = digits_pattern.sub('', s)\n",
        "    # Replace usernames with no space\n",
        "    s = username_pattern.sub('', s)\n",
        "    # Replace hashtags with no space\n",
        "    s = hashtags_pattern.sub('', s)\n",
        "    # Replace <br /> pattern with empty string\n",
        "    s = br_pattern.sub('', s)\n",
        "    # Replace specific characters\n",
        "    s = s.replace(\"https\", \"\")\n",
        "    s = s.replace(\"http\", \"\")\n",
        "    s = s.replace(\"rt\", \"\")\n",
        "    s = s.replace(\"-\", \"\")\n",
        "    # Replace br with empty string\n",
        "    s = s.replace(\"br\", \"\")\n",
        "    # Replace newline character with empty string\n",
        "    s = s.replace(\"\\n\", \"\")\n",
        "    return s"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "7904667e67424983aeacfdf6fceab2a3",
            "5e12365361ab4641a7ba003c86d77586",
            "5db308d5b97f44c48fbedbbf0dbbf97e",
            "8496b3a8f89b420bbd64ba2f7e6fa6f5",
            "4c01bea3d08d4d7b956752b43697c996",
            "68b672b53c8a4a1097f47c2a850ce1a0",
            "12f37aede6d249dd817b16db3b12b099",
            "f7794110d28344d8b713c63aa5ec0e21",
            "6090c48a151a42d5a4daa26662177e7c",
            "fc46c0fc11814ef9bb9045d2dad97234",
            "79887438173e4ad4a55a46e735ec569c"
          ]
        },
        "id": "DyEsn8Lml9PH",
        "outputId": "85cdb938-581e-457e-e341-f575c908f6fb"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Preprocessing:   0%|          | 0/20000 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7904667e67424983aeacfdf6fceab2a3"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from tqdm.notebook import tqdm_notebook\n",
        "\n",
        "preprocessed_reviews = []\n",
        "\n",
        "# Apply preprocessing\n",
        "for review in tqdm_notebook(df['review'], desc='Preprocessing'):\n",
        "    preprocessed_review = preprocess_string(review)\n",
        "    preprocessed_reviews.append(preprocessed_review)\n",
        "\n",
        "# Assign the preprocessed reviews back to  'review' column\n",
        "df['review'] = preprocessed_reviews"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-lwgaucZnAkh",
        "outputId": "9910669d-b29e-42fa-b557-6a37f4d3f8ea"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('The minicupcakes are cute and taste sweet but the vanilla has a strange afteaste from the frosting The black and white is good though Meanwhile their ice creams capn crunch salted caramel cherry vanilla are all atrocious Ive literally never tasted ice cream this bad The salted caramel just tastes like salt the cap n crunch is crappy soggy peanut butter mush and the cherry vanilla just tastes like a bad imitation of Ben and Jerrys The worst by far is not using the berry cap n crunch Holsteins and Momofuku is much better',\n",
              " 'negative')"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "df['review'][0], df['sentiment'][0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BR6tc-oT2Hc1"
      },
      "source": [
        "## 3. Mapping `sentiment` column to numeric values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "vNoB1qL86oPG",
        "outputId": "115f1c9b-11c0-4424-aa2d-86fea5b595c8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              review  sentiment\n",
              "0  The minicupcakes are cute and taste sweet but ...          0\n",
              "1  We had went to the old Satified Frog frequentl...          0\n",
              "2  Very unprofessional front of the house staff T...          0\n",
              "3  We stopped by to check out the happy hour spec...          1\n",
              "4  The pear desse crepe was super delicious The p...          1"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-77fcbbee-a22a-4ac1-a46b-ad831bcbb7cf\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The minicupcakes are cute and taste sweet but ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>We had went to the old Satified Frog frequentl...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Very unprofessional front of the house staff T...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>We stopped by to check out the happy hour spec...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The pear desse crepe was super delicious The p...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-77fcbbee-a22a-4ac1-a46b-ad831bcbb7cf')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-77fcbbee-a22a-4ac1-a46b-ad831bcbb7cf button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-77fcbbee-a22a-4ac1-a46b-ad831bcbb7cf');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-76f172b8-fead-4c39-9d25-bd8e424a171e\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-76f172b8-fead-4c39-9d25-bd8e424a171e')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-76f172b8-fead-4c39-9d25-bd8e424a171e button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 20000,\n  \"fields\": [\n    {\n      \"column\": \"review\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 20000,\n        \"samples\": [\n          \"We went to the Pittsburgh Public Theater last weekend for a show and I was very impressed with the facilities I really like the horse shoe seating the engulfs the stage and the experience of looking down on the actors I also thought the staff was extremely helpful and the actors at the theater were really impressive We had no problem getting in and out ordering a drink quickly at intermission and receiving directions on where to find our seats I like this theater a lot and plan on many return visitsnnThe only small drawback is the seats were incredibly uncomfoable for me Maybe it was because we were in the balcony and had to turn our bodies at a degree angle which would smash your legs again the arm rest or you could just turn your neck but either way it wasnt real comfy I hope in the future we can get a seat more in the middle perhaps alleviating this small issue nnOverall this is just another great addition to the Pittsburghs cultural district\",\n          \"As of today Clark County bureaucrats have raped this park and stolen it from the people nnJust a few days ago New Silver Bowl Park was a acre open grassy field in which you and I were free to run play frisbee play tag football walk your dog play the drums have a picnic But the idea of a unfenced park is intolerable to bureaucrats Today at a cost of  million they are installing menacing  foot high metal spike fencing around the entire park not merely chainlink fencing but the kind of fencing you would expect to find at a prison military base or the Berlin Wall nnAt best if you are physically challenged and cant walk long distances youll have to hike hundreds of yards to find an opening in the fence At worst youll have to use the park at a time and place when your civil masters deign to unchain one of the few openings in the fencennNone of the photos on this page reflect current reality at this park When the fence is complete maybe I will post new ones\",\n          \"Absolutely rude staff unhelpful and seemed irritated by three questions My order was small plain and honestly quite unappealing I was embarrassed when I saw it I will never order from them again To spend  for an undecorated  carnation wrist corsage is absurd I could have made that myself for \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"sentiment\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          1,\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "# Map 'positive' to 1 & 'negative' to 0\n",
        "df['sentiment'] = df['sentiment'].replace({'positive': 1, 'negative': 0})\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9XS665rf2t-9"
      },
      "source": [
        "## 4. Spliiting datasets into train and test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xE28r9W8jFXc",
        "outputId": "04cdb615-93a7-4d78-8b87-fbcccb718c33"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(16000, 4000)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(df['review'],\n",
        "                                                    df['sentiment'],\n",
        "                                                    test_size=0.2)\n",
        "\n",
        "len(X_train), len(X_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "arUMIRZDqr5T",
        "outputId": "8e6e6d00-536d-42da-dfe0-963f990a8abd"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Dont ask me why I love this movie so muchMaybe it came at a time in my life I desperately wanted to fit in maybe it is the amazing monster effects maybe because I enjoyed the novel Cabal but Its probably because I LOVE Clive Barker I think its fair to warn you the movie and the novel have no true resolve and like me youll probably have a WTF moment at the end At least two sequels were planned but never came about due to the fact the movie flopped for a few reasons The studio made drastic cuts to the film cutting a good  or so minutes out of it and they did a HORRIBLE job promoting it The adves made it look like just another cheap slasher showing mainly the Button faceMask Decker character This is a movie about the monsters About fantasy About a place called Midian Its a story where the monsters are the good guys There is truly nothing else out there like it Its not a movie for everybody I suppose but it stands as one of Clives many great works Sit back and be prepared to be taken to Midian where the monsters are',\n",
              "  'awfull poions are small its not too expensive which made me feel happy at the end of the meal cause i would be really mad if i had to spend more than  for that awfull mealnthis place is for old people nothing exciting i had fish sandwich on a very soggy ead sides were very small mac and cheese i had better from the boxnthe food is very southern style not much of a healthy stuff on menu and for sure no choice for vegatarians or health concious maybe thats the reason why everyone thats overweight is therenno we will not be back'],\n",
              " [1, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "X_train, X_test, y_train, y_test = list(X_train), list(X_test), list(y_train), list(y_test)\n",
        "X_train[:2], y_train[:2]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QyrxW-2T20rI"
      },
      "source": [
        "## 5. Preparing data using custom dataloader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sS4L0KVorOIx",
        "outputId": "100cbb56-db5c-4ccf-d2ed-713b178b10cd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset\n",
        "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n",
        "from transformers import Trainer, TrainingArguments\n",
        "\n",
        "# Setting device agnostic code\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "print(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dN_U6_-OtG9V"
      },
      "outputs": [],
      "source": [
        "class data(Dataset):\n",
        "  def __init__(self, encodings, labels):\n",
        "    self.encodings = encodings\n",
        "    self.labels = labels\n",
        "\n",
        "  def __getitem__(self, index):\n",
        "    item = {key: torch.tensor(val[index]) for key, val in self.encodings.items()}\n",
        "    item['labels'] = torch.tensor(self.labels[index])\n",
        "    return item\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.labels)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jIqMd4Kx7h9D"
      },
      "source": [
        "## 6. Load PreTrained BERT Model\n",
        "\n",
        "**BERT** (Bidirectional Encoder Representations from Transformers) is a pre-trained language representation model developed by researchers at Google.\n",
        "\n",
        "<img src = \"https://i0.wp.com/neptune.ai/wp-content/uploads/2022/10/Attention_diagram_transformer.png?ssl=1\">\n",
        "\n",
        "- BERT architecture consists of `multiple encoder transformer blocks` stacked together.\n",
        "- Each transformer block includes` multi-head self-attention` and `feed-forward neural networks`.\n",
        "- `Multi-head self-attention` allows BERT to weigh word importance based on context, capturing long-range dependencies effectively.\n",
        "- The output from `attention mechanisms` undergoes non-linear transformations via `feed-forward neural networks`.\n",
        "- `Layer normalization` and `residual connections` stabilize training and facilitate gradient flow within each transformer block.\n",
        "- `Positional encodings` preserve word order in sequences, aiding BERT in understanding the sequential nature of data.\n",
        "\n",
        ">BERT is pre-trained on a large text corpus using tasks like masked language modeling and next sentence prediction. Fine-tuning on specific tasks involves adjusting the final layers of the pre-trained BERT model.\n",
        "\n",
        "### [Explanation Video on BERT](https://www.youtube.com/watch?v=6ahxPTLZxU8)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import notebook_login\n",
        "\n",
        "# Paste hugging face token with write permission enabled and log in\n",
        "notebook_login()"
      ],
      "metadata": {
        "id": "6WY3qWTAxh1u",
        "outputId": "24259200-a1da-4a6c-f106-4582baa801f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "923e1b7fbd8b49bdabcd4f00f714be14",
            "b809d01b8cd344d98f2273b3ae8f7ffb",
            "6360e06ae7e74b89ad45dd1bb4413342",
            "1421f131fe17439898b48f87cfa26b52",
            "730e7bfb278a433bac4fedce9538cf0d",
            "0cb660d95a9b46ed9ea2689b3b828508",
            "b6caf3716cd4411cb727f34683958bf6",
            "2480db893d3b43379c2aff68f421eb07",
            "e8fd4b2f11b749ca9ef44d94353e13f0",
            "3bc4f28421f8430d8640f292e94d8c64",
            "96edc1fed27340c692bf75fd4fc9c5d2",
            "ee83043d95cc4bee83a14456fbd7c01c",
            "ed42110cf0cf47f3aff0853d0452e7d8",
            "a5c25ef7145d4a819569b30c10b3db22",
            "be723d54691348868cff55a8ba046eab",
            "3242d629683242bfbc5777a2b904dd70",
            "657bc2cc2cb744c4a1f72488b58d53b0",
            "35e38db3a6e74956ae22f6e6b3d594a3",
            "e800a5dc461b465c98814786c7b9d42b",
            "90cea132a5114445922194fa5aea05d1",
            "9f025708b4034c879c0a4249378fadc3",
            "f03f81ec0bc44602be760ff2f548a5fd",
            "038710f43d7a48dc8660458c680b35fb",
            "368cc160a1bb43b7b11f7991b2d55d5a",
            "94bc6056062746bc8b1cfbfa3292a241",
            "6fe80fdd065148a2b8f6eebf0862123a",
            "680bcce08dd645f9badcfd5a9f53dc4f",
            "1d5d3e06619c4084b41ab120e8476c6d",
            "f701050853814fff9c33373ab92deb43",
            "2a4847e7a62f46b0b2afd22f9c1a0a4c",
            "7b0317bfb8d742a09a3aa2c0ebb9773c",
            "b848fa1f848348079650e1897429c3fa"
          ]
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "923e1b7fbd8b49bdabcd4f00f714be14"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M3pUE51Ttyvp",
        "outputId": "f8683f9d-2e89-4c13-8eb5-f311fd46e181",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "model_name = 'distilbert-base-uncased'\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wFFYOaeg--PW"
      },
      "source": [
        "## 7. Tokenize and Create Encoded Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6d1E4UWit4PW"
      },
      "outputs": [],
      "source": [
        "from transformers import DataCollatorWithPadding\n",
        "\n",
        "data_collator = DataCollatorWithPadding(tokenizer)\n",
        "\n",
        "# Tokenize with truncation and padding and create dataset from tokenized data\n",
        "train_encoding = tokenizer(X_train, truncation=True, padding=True)\n",
        "test_encoding = tokenizer(X_test, truncation=True, padding=True)\n",
        "\n",
        "train_dataset = data(train_encoding, y_train)\n",
        "test_dataset = data(test_encoding, y_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zu7-BCBH-wis"
      },
      "source": [
        "## 8. Fine-Tuning Distil BERT\n",
        "\n",
        "Fine-tuning BERT, a pre-trained language model, allows us to adapt it to specific NLP tasks such as text classification, named entity recognition, sentiment analysis, and question answering.\n",
        "\n",
        "\n",
        "<img src = \"https://raw.githubusercontent.com/LuluW8071/Text-Sentiment-Analysis/dfa065d8169ae9d26460114e612118f5628d7dd3//assets/BERT-Fine-tuning-pipeline.png\">"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x4qy_mL05NVG"
      },
      "outputs": [],
      "source": [
        "batch_size = 32\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=f\"{model_name}-finetuned-sentiment\",\n",
        "    num_train_epochs=4,                              # No of epochs to train\n",
        "    per_device_train_batch_size=batch_size,          # Batch size for training per device\n",
        "    per_device_eval_batch_size=batch_size,           # Batch size for evaluation per device\n",
        "    learning_rate=2e-5,                              # Learning rate for optimizer\n",
        "    warmup_steps=400,                                # No of warmup steps for the learning rate scheduler\n",
        "    weight_decay=0.01,                               # Weight decay coefficient for regularization\n",
        "    logging_dir='./logs',                            # Directory for logging training information\n",
        "    load_best_model_at_end=True,                     # Whether to load the best model from checkpoints at the end of training\n",
        "    logging_steps=400,                               # Log training metrics every `logging_steps` steps\n",
        "    save_steps=400,                                  # Save model checkpoints every `save_steps` steps\n",
        "    save_total_limit=2,                              # Save no of checkpoints\n",
        "    eval_strategy = \"steps\",                         # When to run evaluation during training: steps, epochs or none\n",
        "    fp16=True,                                       # Floating point 16 precision\n",
        "    push_to_hub=True,                                # Save checkpoint in Hugging Face Hub\n",
        "    report_to=\"tensorboard\",                         # Enable TensorBoard integration\n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nlH5U0ImAMFJ"
      },
      "source": [
        "## 9. Train the Fine-Tuned BERT Model"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_metric\n",
        "\n",
        "accuracy_metric = load_metric(\"accuracy\")\n",
        "\n",
        "# Compute_metrics function\n",
        "def compute_metrics(eval_pred):\n",
        "    logits, labels = eval_pred\n",
        "    predictions = np.argmax(logits, axis=-1)\n",
        "    accuracy = accuracy_metric.compute(predictions=predictions, references=labels)\n",
        "    return accuracy"
      ],
      "metadata": {
        "id": "0n4eNRbo39LN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QaN-z2lExThi",
        "outputId": "05ed24c8-d387-456b-b27b-0b8046ee9234"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "# Label mapping\n",
        "label_mapping = {0: 'negative', 1: 'positive'}\n",
        "\n",
        "model = DistilBertForSequenceClassification.from_pretrained(model_name,\n",
        "                                                            num_labels=2)\n",
        "\n",
        "# Override the model configuration for custom labels\n",
        "model.config.id2label = label_mapping\n",
        "model.config.label2id = {v: k for k, v in label_mapping.items()}\n",
        "\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,                      # The instantiated Transformers model to be trained\n",
        "    args=training_args,               # Training arguments, defined above\n",
        "    train_dataset=train_dataset,      # Training dataset\n",
        "    eval_dataset=test_dataset,        # Evaluation dataset\n",
        "    tokenizer=tokenizer,              # Tokenizer\n",
        "    data_collator=data_collator,      # Data collator\n",
        "    compute_metrics=compute_metrics,  # Function to compute metrics\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 422
        },
        "id": "H1KmmxIQyXK_",
        "outputId": "dfe4bf31-ce16-4c6f-8aeb-a54dc5a9338b"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1347' max='2000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1347/2000 13:50 < 06:43, 1.62 it/s, Epoch 2.69/4]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>400</td>\n",
              "      <td>0.389300</td>\n",
              "      <td>0.181316</td>\n",
              "      <td>0.932500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>800</td>\n",
              "      <td>0.161900</td>\n",
              "      <td>0.166204</td>\n",
              "      <td>0.941250</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1200</td>\n",
              "      <td>0.114600</td>\n",
              "      <td>0.200135</td>\n",
              "      <td>0.943000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='2000' max='2000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [2000/2000 20:55, Epoch 4/4]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>400</td>\n",
              "      <td>0.389300</td>\n",
              "      <td>0.181316</td>\n",
              "      <td>0.932500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>800</td>\n",
              "      <td>0.161900</td>\n",
              "      <td>0.166204</td>\n",
              "      <td>0.941250</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1200</td>\n",
              "      <td>0.114600</td>\n",
              "      <td>0.200135</td>\n",
              "      <td>0.943000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1600</td>\n",
              "      <td>0.076300</td>\n",
              "      <td>0.211609</td>\n",
              "      <td>0.944000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.041600</td>\n",
              "      <td>0.225439</td>\n",
              "      <td>0.944500</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=2000, training_loss=0.15671546173095702, metrics={'train_runtime': 1257.5696, 'train_samples_per_second': 50.892, 'train_steps_per_second': 1.59, 'total_flos': 8477913513984000.0, 'train_loss': 0.15671546173095702, 'epoch': 4.0})"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ],
      "source": [
        "from accelerate import Accelerator\n",
        "\n",
        "# Initialize Accelerator and Trainer\n",
        "Accelerator()\n",
        "trainer.train()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2wlH9_LX9fG2"
      },
      "source": [
        "## 10. Sentiment Prediction using custom text\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EBo9Vmnh8ET8"
      },
      "outputs": [],
      "source": [
        "# Tokenize text, get output from model and predict\n",
        "def predict_sentiment(model, tokenizer, text, device):\n",
        "    tokenized = tokenizer(text, truncation=True, padding=True, return_tensors='pt').to(device)\n",
        "    outputs = model(**tokenized)\n",
        "\n",
        "    probs = F.softmax(outputs.logits, dim=-1)\n",
        "    preds = torch.argmax(outputs.logits, dim=-1).item()\n",
        "    probs_max = probs.max().detach().cpu().numpy()\n",
        "\n",
        "    prediction = \"Positive\" if preds == 1 else \"Negative\"\n",
        "    print(f'{text}\\nSentiment: {prediction}\\tProbability: {probs_max*100:.2f}%\\n', end=\"-\"*50 + \"\\n\")\n",
        "    # return prediction, probs_max"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EujVy_419ip4"
      },
      "outputs": [],
      "source": [
        "texts = [\n",
        "    \"The traffic was horrendous this morning; I was stuck in it for over an hour.\",\n",
        "    \"I was extremely disappointed with the quality of the product; it didn't meet my expectations at all.\",\n",
        "    \"The customer service at the restaurant was very good; the staff went above and beyond to make us feel welcome.\",\n",
        "    \"My recent stay at Paradise Resort was absolutely fantastic! From the moment I arrived, I was greeted with warm smiles and excellent service. The room was spacious, beautifully decorated, and spotlessly clean. I loved the breathtaking view from my balcony overlooking the pool and tropical gardens. The dining options were exceptional, and the resort's facilities were top-notch, offering everything from a fitness center to guided nature walks. Overall, Paradise Resort exceeded all my expectations, and I can't wait to return for another memorable stay!\",\n",
        "    \"The movie started off promising, but it quickly went downhill. The plot was confusing, the acting was mediocre, and the ending was unsatisfying.\",\n",
        "    \"I had a terrible experience at the restaurant last night. The food was cold, the service was slow, and the staff was rude.\",\n",
        "    \"Despite the initial skepticism, I was pleasantly surprised by the performance of the new smartphone. Its sleek design, impressive camera quality, and fast processing speed exceeded my expectations.\",\n",
        "    \"The concert was absolutely amazing! The energy of the performers, the enthusiasm of the crowd, and the quality of the music made it an unforgettable experience.\",\n",
        "    \"I had high hopes for the book, but it turned out to be a disappointment. The characters were one-dimensional, the plot was predictable, and the writing style was uninspired.\",\n",
        "    \"The presentation was well-prepared and delivered with confidence. The speaker engaged the audience effectively and provided valuable insights on the topic.\",\n",
        "    \"The service at the hotel was impeccable. The staff was attentive, courteous, and always willing to assist with any request.\",\n",
        "    \"The weather during our vacation was dreadful; it rained every day, and we were stuck indoors for most of the trip.\",\n",
        "    \"The hiking trail offered breathtaking views of the mountains and lush forests. It was a challenging but rewarding experience.\",\n",
        "    \"The customer support team was unhelpful and incompetent. They were unable to resolve my issue and seemed indifferent to my concerns.\",\n",
        "    \"The play was a delightful blend of humor, drama, and suspense. The talented cast delivered stellar performances, and the storyline kept me engaged from start to finish.\",\n",
        "    \"The new restaurant in town has quickly become my favorite dining spot. The food is delicious, the atmosphere is cozy, and the service is outstanding.\",\n",
        "]\n",
        "\n",
        "for text in texts:\n",
        "  predict_sentiment(model, tokenizer, text, device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YBC6C8VA_Kh4"
      },
      "outputs": [],
      "source": [
        "# An example of complex review that contains both positive and negative sentiment\n",
        "texts = [\"Despite facing numerous challenges and setbacks, the team worked tirelessly and managed to exceed all expectations, achieving remarkable success. However, despite their best efforts, the project encountered multiple setbacks, ultimately leading to its failure and significant financial losses.\",\n",
        "         \"The hotel room was clean and comfortable, and the amenities were well-maintained. However, the noise from the nearby construction site was disruptive due to which i could not focus when working.\",\n",
        "         \"The movie had an intriguing plot and captivating visuals, but the sound quality was poor, making it difficult to fully enjoy the experience.\"]\n",
        "for text in texts:\n",
        "  predict_sentiment(model, tokenizer, text, device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ucfdIBfD62Tr"
      },
      "outputs": [],
      "source": [
        "# Breaking down above example into parts\n",
        "texts = [\"Despite facing numerous challenges and setbacks, the team worked tirelessly and managed to exceed all expectations, achieving remarkable success.\",\n",
        "         \"However, despite their best efforts, the project encountered multiple setbacks, ultimately leading to its failure and significant financial losses.\",\n",
        "         \"The hotel room was clean and comfortable, and the amenities were well-maintained.\",\n",
        "         \"However, the noise from the nearby construction site was disruptive due to which i could not focus when working.\"]\n",
        "\n",
        "for text in texts:\n",
        "  predict_sentiment(model, tokenizer, text, device)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Looks like **BERT** can accurately interpret the overall sentiment of the text, recognizing the positive aspects (clean and comfortable room, well-maintained amenities) as well as the negative aspect (disruptive noise from construction). By considering the context and weighing the various sentiments present, BERT can provide a nuanced understanding of the text's sentiment.\n",
        "\n",
        "Overall, BERT's capability to handle mixed sentiments reflects its robustness and versatility in natural language understanding, making it a valuable tool for sentiment analysis and various other NLP tasks."
      ],
      "metadata": {
        "id": "zH1RF9aR5okZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 11. Evaluate & Plot Confusion Matrix"
      ],
      "metadata": {
        "id": "i30fs6fub2qs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "from tqdm.auto import tqdm\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "2vhoNPwXYyIY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Predict X_test dataset and evaluate through usage of metrics and\n",
        "# plot Confusion Matrix\n",
        "def predict_sentiment_and_evaluate(model, tokenizer, X_test, y_test, device):\n",
        "  predictions = []\n",
        "\n",
        "  for text in tqdm(X_test):\n",
        "    # Tokenize and forward pass to model\n",
        "    tokenized = tokenizer(text, truncation=True, padding=True, return_tensors='pt').to(device)\n",
        "    outputs = model(**tokenized)\n",
        "\n",
        "    # Inference\n",
        "    preds = torch.argmax(outputs.logits, dim=-1).item()\n",
        "    prediction = 1 if preds == 1 else 0\n",
        "    predictions.append(prediction)\n",
        "\n",
        "  # Confusion Matrix\n",
        "  cm = confusion_matrix(y_test, predictions)\n",
        "  plt.figure(figsize=(6, 4))\n",
        "  sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\", cbar=False,\n",
        "              xticklabels=['Negative', 'Positive'],\n",
        "              yticklabels=['Negative', 'Positive'])\n",
        "  plt.xlabel('Predicted')\n",
        "  plt.ylabel('Actual')\n",
        "  plt.title('Confusion Matrix')\n",
        "  plt.show()"
      ],
      "metadata": {
        "id": "Tw25ZE-BbwY8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predict_sentiment_and_evaluate(model, tokenizer, X_test, y_test, device)"
      ],
      "metadata": {
        "id": "6LimqTJ9YzCG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442,
          "referenced_widgets": [
            "651b42c5bcb446e58ee60d07411982cf",
            "e2c20c8499e1439c83e12333b6876280",
            "47d938799f6149bca0349e65366cad51",
            "24b53ee1f6b44eb996574e5c8d9231b8",
            "3f1d4ce564974fa5be80e15978afb915",
            "095ce34a8efb4ebab6df1faf3ab9257a",
            "7f556609234146d08c78c0bbea32ec93",
            "a71b7b35ac20470aa715857fe4eac0d3",
            "4cb94bfc2fc94e8b8787f41612b7f749",
            "54f7632a7d3047ce9b6e9333ac871248",
            "198d620b60f14cd2aa5cfb0114dec311"
          ]
        },
        "outputId": "b30b98dd-c38c-40a0-9968-9e460bb164ea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/4000 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "651b42c5bcb446e58ee60d07411982cf"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 600x400 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhAAAAGJCAYAAADbgQqfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6vUlEQVR4nO3deZxO5f/H8fc92z2bGWOZ7DPDMExZopIlQ02NNUsloWYkW4psoe+3QiHKNioi2RqlkGwVsiVLi7FkXwZhZMk2hlnP7w8/99dtZpijGfctr+fj4fEw17nOdT7nrpu361znHIthGIYAAABMcHF0AQAA4M5DgAAAAKYRIAAAgGkECAAAYBoBAgAAmEaAAAAAphEgAACAaQQIAABgGgECAACYRoAA7gJ79+7VE088IX9/f1ksFs2fPz9Pxz948KAsFoumTZuWp+PeyerXr6/69es7ugwg3xAggNtk//796tKli8qWLStPT0/5+fmpTp06GjdunC5dupSvx46Ojta2bds0dOhQzZw5Uw888EC+Hu92iomJkcVikZ+fX7af4969e2WxWGSxWPTBBx+YHv/YsWMaNGiQNm/enAfVAv8ebo4uALgbLF68WM8884ysVqteeOEF3XfffUpNTdXatWvVr18/bd++XZMmTcqXY1+6dEnr16/Xf/7zH73yyiv5coygoCBdunRJ7u7u+TL+zbi5uSk5OVkLFy5U69at7bbFxcXJ09NTly9fvqWxjx07psGDBys4OFjVqlXL9X5Lly69peMBdwoCBJDPEhIS1KZNGwUFBWnFihUqXry4bVv37t21b98+LV68ON+Of/LkSUlSwYIF8+0YFotFnp6e+Tb+zVitVtWpU0dffPFFlgAxa9YsNWnSRHPnzr0ttSQnJ8vb21seHh635XiAo3AJA8hnI0eOVFJSkqZMmWIXHq4KDQ1Vz549bT+np6frnXfeUbly5WS1WhUcHKw33nhDKSkpdvsFBweradOmWrt2rR566CF5enqqbNmymjFjhq3PoEGDFBQUJEnq16+fLBaLgoODJV2Z+r/6+2sNGjRIFovFrm3ZsmWqW7euChYsKF9fX4WFhemNN96wbc9pDcSKFSv0yCOPyMfHRwULFlTz5s21c+fObI+3b98+xcTEqGDBgvL391eHDh2UnJyc8wd7nbZt2+q7777T2bNnbW2//vqr9u7dq7Zt22bp//fff6tv376qXLmyfH195efnp0aNGmnLli22PqtWrdKDDz4oSerQoYPtUsjV86xfv77uu+8+/f7776pXr568vb1tn8v1ayCio6Pl6emZ5fyjoqIUEBCgY8eO5fpcAWdAgADy2cKFC1W2bFnVrl07V/1feuklvfXWW6pevbrGjBmjiIgIDR8+XG3atMnSd9++fXr66af1+OOPa9SoUQoICFBMTIy2b98uSWrVqpXGjBkjSXruuec0c+ZMjR071lT927dvV9OmTZWSkqIhQ4Zo1KhRevLJJ/Xzzz/fcL/ly5crKipKJ06c0KBBg9S7d2+tW7dOderU0cGDB7P0b926tS5cuKDhw4erdevWmjZtmgYPHpzrOlu1aiWLxaJ58+bZ2mbNmqWKFSuqevXqWfofOHBA8+fPV9OmTTV69Gj169dP27ZtU0REhO0v80qVKmnIkCGSpM6dO2vmzJmaOXOm6tWrZxvn9OnTatSokapVq6axY8eqQYMG2dY3btw4FS1aVNHR0crIyJAkffLJJ1q6dKnGjx+vEiVK5PpcAadgAMg3586dMyQZzZs3z1X/zZs3G5KMl156ya69b9++hiRjxYoVtragoCBDkrFmzRpb24kTJwyr1Wr06dPH1paQkGBIMt5//327MaOjo42goKAsNbz99tvGtX80jBkzxpBknDx5Mse6rx5j6tSptrZq1aoZgYGBxunTp21tW7ZsMVxcXIwXXnghy/FefPFFuzFbtmxpFC5cOMdjXnsePj4+hmEYxtNPP2089thjhmEYRkZGhlGsWDFj8ODB2X4Gly9fNjIyMrKch9VqNYYMGWJr+/XXX7Oc21URERGGJGPixInZbouIiLBr++GHHwxJxrvvvmscOHDA8PX1NVq0aHHTcwScETMQQD46f/68JKlAgQK56r9kyRJJUu/eve3a+/TpI0lZ1kqEh4frkUcesf1ctGhRhYWF6cCBA7dc8/Wurp349ttvlZmZmat9EhMTtXnzZsXExKhQoUK29ipVqujxxx+3nee1unbtavfzI488otOnT9s+w9xo27atVq1apePHj2vFihU6fvx4tpcvpCvrJlxcrvwRmJGRodOnT9suz2zatCnXx7RarerQoUOu+j7xxBPq0qWLhgwZolatWsnT01OffPJJro8FOBMCBJCP/Pz8JEkXLlzIVf9Dhw7JxcVFoaGhdu3FihVTwYIFdejQIbv2MmXKZBkjICBAZ86cucWKs3r22WdVp04dvfTSS7rnnnvUpk0bffXVVzcME1frDAsLy7KtUqVKOnXqlC5evGjXfv25BAQESJKpc2ncuLEKFCig2bNnKy4uTg8++GCWz/KqzMxMjRkzRuXLl5fValWRIkVUtGhRbd26VefOncv1MUuWLGlqweQHH3ygQoUKafPmzYqNjVVgYGCu9wWcCQECyEd+fn4qUaKE/vjjD1P7Xb+IMSeurq7ZthuGccvHuHp9/iovLy+tWbNGy5cv1/PPP6+tW7fq2Wef1eOPP56l7z/xT87lKqvVqlatWmn69On65ptvcpx9kKRhw4apd+/eqlevnj7//HP98MMPWrZsme69995cz7RIVz4fM+Lj43XixAlJ0rZt20ztCzgTAgSQz5o2bar9+/dr/fr1N+0bFBSkzMxM7d271679r7/+0tmzZ213VOSFgIAAuzsWrrp+lkOSXFxc9Nhjj2n06NHasWOHhg4dqhUrVmjlypXZjn21zt27d2fZtmvXLhUpUkQ+Pj7/7ARy0LZtW8XHx+vChQvZLjy9as6cOWrQoIGmTJmiNm3a6IknnlBkZGSWzyS3YS43Ll68qA4dOig8PFydO3fWyJEj9euvv+bZ+MDtRIAA8tnrr78uHx8fvfTSS/rrr7+ybN+/f7/GjRsn6coUvKQsd0qMHj1aktSkSZM8q6tcuXI6d+6ctm7damtLTEzUN998Y9fv77//zrLv1QcqXX9r6VXFixdXtWrVNH36dLu/kP/44w8tXbrUdp75oUGDBnrnnXf04YcfqlixYjn2c3V1zTK78fXXX+vo0aN2bVeDTnZhy6z+/fvr8OHDmj59ukaPHq3g4GBFR0fn+DkCzowHSQH5rFy5cpo1a5aeffZZVapUye5JlOvWrdPXX3+tmJgYSVLVqlUVHR2tSZMm6ezZs4qIiNAvv/yi6dOnq0WLFjneIngr2rRpo/79+6tly5bq0aOHkpOTNWHCBFWoUMFuEeGQIUO0Zs0aNWnSREFBQTpx4oQ+/vhjlSpVSnXr1s1x/Pfff1+NGjVSrVq11LFjR126dEnjx4+Xv7+/Bg0alGfncT0XFxf997//vWm/pk2basiQIerQoYNq166tbdu2KS4uTmXLlrXrV65cORUsWFATJ05UgQIF5OPjo5o1ayokJMRUXStWrNDHH3+st99+23Zb6dSpU1W/fn29+eabGjlypKnxAIdz8F0gwF1jz549RqdOnYzg4GDDw8PDKFCggFGnTh1j/PjxxuXLl2390tLSjMGDBxshISGGu7u7Ubp0aWPgwIF2fQzjym2cTZo0yXKc628fzOk2TsMwjKVLlxr33Xef4eHhYYSFhRmff/55lts4f/zxR6N58+ZGiRIlDA8PD6NEiRLGc889Z+zZsyfLMa6/1XH58uVGnTp1DC8vL8PPz89o1qyZsWPHDrs+V493/W2iU6dONSQZCQkJOX6mhmF/G2dOcrqNs0+fPkbx4sUNLy8vo06dOsb69euzvf3y22+/NcLDww03Nze784yIiDDuvffebI957Tjnz583goKCjOrVqxtpaWl2/Xr16mW4uLgY69evv+E5AM7GYhgmVigBAACINRAAAOAWECAAAIBpBAgAAGAaAQIAAJhGgAAAAKYRIAAAgGkECAAAYNq/8kmUXlEfOLoEADeQOL+Xo0sAkIOCXtm/2O56zEAAAADTCBAAAMA0AgQAADCNAAEAAEwjQAAAANMIEAAAwDQCBAAAMI0AAQAATCNAAAAA0wgQAADANAIEAAAwjQABAABMI0AAAADTCBAAAMA0AgQAADCNAAEAAEwjQAAAANMIEAAAwDQCBAAAMI0AAQAATCNAAAAA0wgQAADANAIEAAAwjQABAABMI0AAAADTCBAAAMA0AgQAADCNAAEAAEwjQAAAANMIEAAAwDQCBAAAMI0AAQAATCNAAAAA0wgQAADANAIEAAAwjQABAABMI0AAAADTCBAAAMA0AgQAADCNAAEAAEwjQAAAANMIEAAAwDQCBAAAMI0AAQAATCNAAAAA0wgQAADANAIEAAAwjQABAABMI0AAAADTnCZA/PTTT2rfvr1q1aqlo0ePSpJmzpyptWvXOrgyAABwPacIEHPnzlVUVJS8vLwUHx+vlJQUSdK5c+c0bNgwB1cHAACu5xQB4t1339XEiRM1efJkubu729rr1KmjTZs2ObAyAACQHacIELt371a9evWytPv7++vs2bO3vyAAAHBDThEgihUrpn379mVpX7t2rcqWLeuAigAAwI04RYDo1KmTevbsqY0bN8pisejYsWOKi4tT37591a1bN0eXBwAAruPm6AIkacCAAcrMzNRjjz2m5ORk1atXT1arVX379tWrr77q6PIAAMB1LIZhGI4u4qrU1FTt27dPSUlJCg8Pl6+v7y2N4xX1QR5XBiAvJc7v5egSAOSgoJdrrvo5xSWMzz//XMnJyfLw8FB4eLgeeuihWw4PAAAg/zlFgOjVq5cCAwPVtm1bLVmyRBkZGY4uCQAA3IBTBIjExER9+eWXslgsat26tYoXL67u3btr3bp1ji4NAABkwykChJubm5o2baq4uDidOHFCY8aM0cGDB9WgQQOVK1fO0eUBAIDrOMVdGNfy9vZWVFSUzpw5o0OHDmnnzp2OLgkAAFzHKWYgJCk5OVlxcXFq3LixSpYsqbFjx6ply5bavn27o0sDAADXcYoZiDZt2mjRokXy9vZW69at9eabb6pWrVqOLgsAAOTAKQKEq6urvvrqK0VFRcnVNXf3nwIAAMdxigARFxfn6BIAAIAJDgsQsbGx6ty5szw9PRUbG3vDvj169LhNVQEAgNxw2KOsQ0JC9Ntvv6lw4cIKCQnJsZ/FYtGBAwdMjc2jrAHnxqOsAeeV20dZO2wGIiEhIdvfAwAA5+cUt3EOGTJEycnJWdovXbqkIUOGOKAiAABwI07xNk5XV1clJiYqMDDQrv306dMKDAw0/W4MLmE4lzr3lVKvZx5U9fL3qHhhX7UeNF8L1++zbffxdNe7HeupWa1QFfLz1MHj5/Xxt5v06eIttj5Wd1e917m+nqlfUVZ3Vy3//aB6jl+uE2evBM/KZYuqb+uHVPu+kirs56VDf53Xp4u36KP5m277+eLmuITh3OJ//02fT/9Mu3Zu16mTJzVydKwiHo20bR/y5htavHC+3T4P166rcR9Psv3colGkEhOP2fV5uUcvRb/YKV9rxz/n9JcwrmUYhiwWS5b2LVu2qFChQg6oCHnJx9Nd2w6c0Iwftmn22y2ybB/Rpb7qVyujDiOX6NBf5xRZPVjjXo1U4ukkLd6wX5I0smsDNXqorNq9u0DnL6ZqTPfH9OVbzfVo7y8kSfeH3qOTZ5PVYcQSHTl5QQ+Hl9BHPZ9QRqahiQvib+fpAne8S5eSVb5CmJq1aKX+vbNfxF6rTl29OXio7Wd3D48sfTq//KpatHra9rO3j0/eFwuHcWiACAgIkMVikcViUYUKFexCREZGhpKSktS1a1cHVoi8sPS3BC39Led1Lg+Hl9Tny7brp61/SpI++26rOjapogfCimnxhv3y8/ZQTFRlxby3WKu3XOnTefT32vLpi3qoYnH9sitRM5b+YTfmwePnVLNSCTWvU54AAZhUu2491a5b74Z93N09VLhI0Rv28fb2uWkf3LkcGiDGjh0rwzD04osvavDgwfL397dt8/DwUHBwME+kvAts2HFUTR8O1Ywf/tCx00mqV7W0ypcspNcnrpIk3V/+Hnm4u2pF/CHbPnv+/FuH/zqvmpVK6JddidmO6+9j1ZkLl2/HKQB3nU2//aqGDeqqgJ+fHnioprp27yn/ggXt+syYOlmfTZ6gYsWK64lGTfRc+2i5uTnFxDfygEP/S0ZHR0u6cktn7dq15e7u7shy4CC9P16hj3o+of2zuiotPUOZmYZeHrdUP/9xRJJUrJCPUlLTde5iit1+J85e1D2Fsp8SfTi8hJ6OCFPLN+fle/3A3ebhOnVV/7FIlShZSkf/PKyPPxyr17p30aczZtmeJty6bXuFVQyXn7+/tm2J18exY3X61Cm91re/g6tHXnGKKBgREWH7/eXLl5Wammq33c/PL8d9U1JSlJJi/xeLkZkui4tTnBpy4eXm9+uhisX11FvzdPjEedWtXFpju19ZA7Ey/rDp8cKDiuirt1to6Ofr9eOmQzffAYApTzRsbPt9aPkKCq0QplZNo7Tpt1/0YM0rs8Ztn4+x9SlfIUxu7u56793BerlHL3lks14Cdx6nuI0zOTlZr7zyigIDA+Xj46OAgAC7XzcyfPhw+fv72/1KP7DiNlWOf8rTw02DYx5R/0mrtGTjAf2RcEoTF8Rrzupdeu3pByVJx/++KKuHm/x9rHb7Bhb00V9/X7Rrq1imsJaMeEaffbdVI77YcNvOA7iblSxVWgUDAvTnnzkH/vvuq6KM9HQlHjt6GytDfnKKANGvXz+tWLFCEyZMkNVq1aeffqrBgwerRIkSmjFjxg33HThwoM6dO2f3y63so7epcvxT7m4u8nB3VWam/d3EGZmGXP5/UW383r+UmpahBveXsW0vXypAZe7x08ad/7tNrFJQYX0/srXilm3XoGlrb88JANBffx3XubNnVeQGCyb37N4lFxcXBXBn3b+GU8zzL1y4UDNmzFD9+vXVoUMHPfLIIwoNDVVQUJDi4uLUrl27HPe1Wq2yWu3/ZcrlC+fi4+muciUK2n4OLuavKmWL6syFy/rz5AWt2fKnhnWK0KXUdB3+67weqVJK7SLD1X/SKknS+eRUTfthm0Z0bqC/L1zWhYupGt39UW3YcdS2gDI8qIi+G9lay39LUOy833RPgLekK0Hk1LlLt/uUgTtacvJFHTn8v9mEY0ePas+unfLz95efv78+nfixGkQ+ocKFi+jokcMaP3aUSpUuo4dr15UkbduyWX9s26oaDz4kHx8fbduyWWM/GKGGjZvJz88/p8PiDuMUD5Ly9fXVjh07VKZMGZUqVUrz5s3TQw89pISEBFWuXFlJSUmmxuNBUs7lkSqltfT9Z7O0z1z6hzqP+l73BHhryIv1FFk9SAEFPHX4xHl9tmSrYuf9but79UFSrRtUlNXdTct/S1DPD5frrzNXHiT1n/a19d/na2c5xqHj51QxenL+nRxuCQ+Scm6///qLXu4Uk6W9SbMWev0/b+n1Xq9qz66dunDhvIoWDdRDteqoS/dXVbhwEUnSrp07NHLYEB1KSFBaWqqKlyypRk2eVNvnY1j/cAfI7YOknCJAVKlSRePHj1dERIQiIyNVrVo1ffDBB4qNjdXIkSN15MgRU+MRIADnRoAAnFduA4RTrIHo0KGDtmy58tjiAQMG6KOPPpKnp6d69eqlfv36Obg6AABwPaeYgbjeoUOH9Pvvvys0NFRVqlQxvT8zEIBzYwYCcF531LswrhcUFKSgoCBHlwEAAHLgFAEiNjY223aLxSJPT0+FhoaqXr16tiecAQAAx3KKADFmzBidPHlSycnJtgdHnTlzRt7e3vL19dWJEydUtmxZrVy5UqVLl3ZwtQAAwCkWUQ4bNkwPPvig9u7dq9OnT+v06dPas2ePatasqXHjxunw4cMqVqyYevXiuikAAM7AKRZRlitXTnPnzlW1atXs2uPj4/XUU0/pwIEDWrdunZ566iklJmb/5sVrsYgScG4sogSc1x11G2diYqLS09OztKenp+v48eOSpBIlSujChQu3uzQAAJANpwgQDRo0UJcuXRQfH29ri4+PV7du3fToo1fea7Ft2zaFhIQ4qkQAAHANpwgQU6ZMUaFChVSjRg3buy0eeOABFSpUSFOmTJF05XHXo0aNcnClAABAcpI1EFft2rVLe/bskSSFhYUpLCzslsZhDQTg3FgDATivO/JBUmXLlpXFYlG5cuXk5uZUpQEAgGs4xSWM5ORkdezYUd7e3rr33nt1+P9fI/vqq6/qvffec3B1AADgek4RIAYOHKgtW7Zo1apV8vT0tLVHRkZq9uzZDqwMAABkxymuE8yfP1+zZ8/Www8/LIvFYmu/9957tX//fgdWBgAAsuMUMxAnT55UYGBglvaLFy/aBQoAAOAcnCJAPPDAA1q8eLHt56uh4dNPP1WtWrUcVRYAAMiBU1zCGDZsmBo1aqQdO3YoPT1d48aN044dO7Ru3TqtXr3a0eUBAIDrOMUMRN26dbV582alp6ercuXKWrp0qQIDA7V+/XrVqFHD0eUBAIDrOMUMhHTlhVqTJ092dBkAACAXHBogXFxcbrpI0mKxZPuiLQAA4DgODRDffPNNjtvWr1+v2NhYZWZm3saKAABAbjg0QDRv3jxL2+7duzVgwAAtXLhQ7dq105AhQxxQGQAAuBGnWEQpSceOHVOnTp1UuXJlpaena/PmzZo+fbqCgoIcXRoAALiOwwPEuXPn1L9/f4WGhmr79u368ccftXDhQt13332OLg0AAOTAoZcwRo4cqREjRqhYsWL64osvsr2kAQAAnI/FMAzDUQd3cXGRl5eXIiMj5eqa8/vH582bZ2pcr6gP/mlpAPJR4vxeji4BQA4KeuX89/G1HDoD8cILL/CuCwAA7kAODRDTpk1z5OEBAMAtcvgiSgAAcOchQAAAANMIEAAAwDQCBAAAMI0AAQAATCNAAAAA0wgQAADANAIEAAAwjQABAABMI0AAAADTCBAAAMA0AgQAADCNAAEAAEwjQAAAANMIEAAAwDQCBAAAMI0AAQAATCNAAAAA0wgQAADANAIEAAAwjQABAABMI0AAAADTCBAAAMA0AgQAADDNLTedFixYkOsBn3zyyVsuBgAA3BlyFSBatGiRq8EsFosyMjL+ST0AAOAOkKsAkZmZmd91AACAOwhrIAAAgGm5moG43sWLF7V69WodPnxYqampdtt69OiRJ4UBAADnZTpAxMfHq3HjxkpOTtbFixdVqFAhnTp1St7e3goMDCRAAABwFzB9CaNXr15q1qyZzpw5Iy8vL23YsEGHDh1SjRo19MEHH+RHjQAAwMmYDhCbN29Wnz595OLiIldXV6WkpKh06dIaOXKk3njjjfyoEQAAOBnTAcLd3V0uLld2CwwM1OHDhyVJ/v7++vPPP/O2OgAA4JRMr4G4//779euvv6p8+fKKiIjQW2+9pVOnTmnmzJm677778qNGAADgZEzPQAwbNkzFixeXJA0dOlQBAQHq1q2bTp48qUmTJuV5gQAAwPmYnoF44IEHbL8PDAzU999/n6cFAQAA58eDpAAAgGmmZyBCQkJksVhy3H7gwIF/VBAAAHB+pgPEa6+9ZvdzWlqa4uPj9f3336tfv355VRcAAHBipgNEz549s23/6KOP9Ntvv/3jggAAgPPLszUQjRo10ty5c/NqOAAA4MTyLEDMmTNHhQoVyqvhAACAE7ulB0ldu4jSMAwdP35cJ0+e1Mcff5ynxQEAAOdkMQzDMLPDoEGD7AKEi4uLihYtqvr166tixYp5XuCtuJzu6AoA3EjAg684ugQAObgU/2Gu+pkOEHcCAgTg3AgQgPPKbYAwvQbC1dVVJ06cyNJ++vRpubq6mh0OAADcgUwHiJwmLFJSUuTh4fGPCwIAAM4v14soY2NjJUkWi0WffvqpfH19bdsyMjK0Zs0ap1kDAQAA8leuA8SYMWMkXZmBmDhxot3lCg8PDwUHB2vixIl5XyEAAHA6uQ4QCQkJkqQGDRpo3rx5CggIyLeiAACAczP9HIiVK1fmRx0AAOAOYnoR5VNPPaURI0ZkaR85cqSeeeaZPCkKAAA4N9MBYs2aNWrcuHGW9kaNGmnNmjV5UhQAAHBupgNEUlJStrdruru76/z583lSFAAAcG6mA0TlypU1e/bsLO1ffvmlwsPD86QoAADg3EwvonzzzTfVqlUr7d+/X48++qgk6ccff9SsWbM0Z86cPC8QAAA4H9MBolmzZpo/f76GDRumOXPmyMvLS1WrVtWKFSt4nTcAAHeJf/wyrfPnz+uLL77QlClT9PvvvysjIyOvartlvEwLcG68TAtwXvn2Mq2r1qxZo+joaJUoUUKjRo3So48+qg0bNtzqcAAA4A5i6hLG8ePHNW3aNE2ZMkXnz59X69atlZKSovnz57OAEgCAu0iuZyCaNWumsLAwbd26VWPHjtWxY8c0fvz4/KwNAAA4qVzPQHz33Xfq0aOHunXrpvLly+dnTQAAwMnlegZi7dq1unDhgmrUqKGaNWvqww8/1KlTp/KzNgAA4KRyHSAefvhhTZ48WYmJierSpYu+/PJLlShRQpmZmVq2bJkuXLiQn3UCAAAn8o9u49y9e7emTJmimTNn6uzZs3r88ce1YMGCvKzvlnAbJ+DcuI0TcF75fhunJIWFhWnkyJE6cuSIvvjii38yFAAAuIP84wdJOSNmIADnxgwE4LxuywwEAAC4OxEgAACAaQQIAABgGgECAACYRoAAAACmESAAAIBpBAgAAGAaAQIAAJhGgAAAAKYRIAAAgGkECAAAYBoBAgAAmEaAAAAAphEgAACAaQQIAABgGgECAACYRoAAAACmESAAAIBpBAgAAGAaAQIAAJhGgAAAAKYRIAAAgGkECAAAYBoBAgAAmEaAAAAAphEgAACAaQQIAABgGgECAACYRoAAAACmESAAAIBpThMgfvrpJ7Vv3161atXS0aNHJUkzZ87U2rVrHVwZAAC4nlMEiLlz5yoqKkpeXl6Kj49XSkqKJOncuXMaNmyYg6sDAADXc4oA8e6772rixImaPHmy3N3dbe116tTRpk2bHFgZAADIjlMEiN27d6tevXpZ2v39/XX27NnbXxAAALghpwgQxYoV0759+7K0r127VmXLlnVARQAA4EacIkB06tRJPXv21MaNG2WxWHTs2DHFxcWpb9++6tatm6PLAwAA13FzdAGSNGDAAGVmZuqxxx5TcnKy6tWrJ6vVqr59++rVV191dHkAAOA6FsMwDEcXcVVqaqr27dunpKQkhYeHy9fX95bGuZyex4UByFMBD77i6BIA5OBS/Ie56ucUlzA+//xzJScny8PDQ+Hh4XrooYduOTwAAID85xQBolevXgoMDFTbtm21ZMkSZWRkOLokAABwA04RIBITE/Xll1/KYrGodevWKl68uLp3765169Y5ujQAAJANp1oDIUnJycn65ptvNGvWLC1fvlylSpXS/v37TY3BGgjAubEGAnBeuV0D4RR3YVzL29tbUVFROnPmjA4dOqSdO3c6uiQAAHAdp7iEIV2ZeYiLi1Pjxo1VsmRJjR07Vi1bttT27dsdXRoAALiOU8xAtGnTRosWLZK3t7dat26tN998U7Vq1XJ0WQAAIAdOESBcXV311VdfKSoqSq6uro4uBwAA3IRTBIi4uDhHlwAAAExwWICIjY1V586d5enpqdjY2Bv27dGjx22qCo6SkZGhCR+N1+JFC3T61CkVDQzUk81bqnPXl2WxWCRJhmHo4w9jNW/O17pw4byq3V9d/3lrkIKCgh1bPHCHq1O9nHq9EKnq4WVUvKi/WveapIWrttq2BxYqoHd7NldkrUry9/XS2k371Hvk19p/+KTdODWrhGhQ96Z6sHKwMjIytXXPUTV7+SNdTkmTJH09touqViipooUK6Mz5ZK3cuFv/jf1WiSfP3dbzRd5w2G2cISEh+u2331S4cGGFhITk2M9isejAgQOmxuY2zjvPp5Mmaub0qXpn2AiVCw3Vjj/+0Fv/HahXevZSu/YvSJI++3SSPvt0kt4Z9p5Kliylj8aP0969e/TNgiWyWq0OPgOYwW2czuWJOuGqVbWs4nce1uzRnbMEiFXT+ygtPUMDRs3T+YuX1aP9o3qiTrjub/Wuki+nSroSHr798GV9MHWpFq/epvSMTFWpUFILV21TatqVP5RfbddAG7cm6PipcyoRWFDDe7WUJDWIGX37Txo5cvrbOBMSErL9Pe5OmzfHq/6jj6leRH1JUsmSpfTdksX6Y9uVP8QMw1DczBnq1KWbGjwaKUl6d/hIPVqvtlb8uFyNGjdxVOnAHW/pzzu09Ocd2W4LLROomlVCVP2pd7XzwHFJUo9hs3Vw+TC1blRD075ZL0ka2aeVPv5ylT6Yusy2795DJ+zGGh+30vb7w4ln9MHUZfpqdCe5ubkoPT0zr08L+cwpbuMcMmSIkpOTs7RfunRJQ4YMcUBFuN2qVbtfv2zYoIMHr4TJ3bt2KT7+d9V9pJ4k6eiRIzp16qRqPlzbtk+BAgVUuUpVbd0S75CagbuB1ePKvzMvp/5vatcwDKWmpqt2tXKSpKIBvnqoSohO/p2kldN66+DyYVr6aU/VrlY2x3ED/LzVptED2rAlgfBwh3KKADF48GAlJSVlaU9OTtbgwYMdUBFutxdf6qyoRo3Vomkj1ah6r559uoXaPx+tJk2flCSdOnXlWmvhIoXt9itcuLBOnTp12+sF7ha7Dx7X4cS/9c6rT6pgAS+5u7mqT0ykShULULEi/pKkkFJFJEn/6dJYn81bp+bdP9bmnX9qySevqlyZonbjvdujuU6tG6Vjq0eqdPFCeqbXpNt+TsgbThEgDMOwLZS71pYtW1SoUKEb7puSkqLz58/b/UpJScmvUpFPfvj+Oy1ZvFDDR47Sl1/P0zvD3tP0qZ9pwfxvHF0acFdLT89Umz6TFRoUqMQ17+vv9aNV74EK+n7tdmUaV2YOXFyu/Pk9Ze5azVywQVt2H9Hro+Zpz8ETim5u/0yfMTOW6+E2I9Sk64fKyMjUp+88f9vPCXnDobdxBgQEyGKxyGKxqEKFCnYhIiMjQ0lJSeratesNxxg+fHiWWYr/vPm2/vvWoPwoGflkzKiRerFjZ9tahvIVwpR47JimfPqJnmzRUkWKXPlXzOlTp1W0aKBtv9OnTyusYkWH1AzcLeJ3/qmH27wnP19Pebi76dSZJK2Z0Ve/7zgsSUo8eV6SbGskrtqdcFyliwXYtZ0+e1Gnz17UvsMntDvhuPb98K5qVgnRxq2shbvTODRAjB07VoZh6MUXX9TgwYPl7+9v2+bh4aHg4OCbPpFy4MCB6t27t12b4cqK/DvN5UuXbf+KucrV1VWZmVduEipZqpSKFCmqjRvXq2KlSpKkpKQkbdu6Rc88+9xtrxe4G51PuixJKlemqKqHl9HgjxdJkg4dO61jJ86qQnCgXf/QoMAcF2dK/5u58HB3ikcSwSSH/leLjo6WdOWWztq1a8vd3d30GFarNcstfNzGeeeJqN9AkydNVLHiJVQuNFS7du7UzOlT1bzlU5Ku3M7b7vkXNPmTCQoqE6SSpa7cxlk0MFCPPhbp4OqBO5uPl4fKlf7fWoXgkoVVpUJJnTmfrD+Pn1GryPt18kyS/jz+t+4rX0If9HtaC1dt1Y8bdtn2GTN9uf7btYm27TmqLbuPqH2zmgoLvkdt+02RJD14X5Bq3BukdfH7dfZCskJKFdXbLzfR/sMnmX24QznsORDnz5+Xn5+f7fc3crVfbhEg7jwXLybpo9hxWvHjcv3992kVDQxUo0ZN1KVbd7l7eEj634Ok5n79lS5cOK/7q9fQG2++reDgnJ8jAufEcyCcyyM1ymvppz2ztM9csEGd3/5cLz8XoV4vRCqwcAEdP3VecYs2avik75WWnmHXv2+Hx9WldT0F+Htr256j+s/Y+Vq3+cpzfO4NLaEP+j2lyhVKycfLQ8dPndPSdTs1YvL3OsaDpJxKbp8D4bAA4erqqsTERAUGBsrFxSXbRZRXF1dmZGRkM0LOCBCAcyNAAM7L6R8ktWLFCtsdFitXrrxJbwAA4EwcNgORn5iBAJwbMxCA88rtDIRTPAfi+++/19q1a20/f/TRR6pWrZratm2rM2fOOLAyAACQHacIEP369bMtpNy2bZt69+6txo0bKyEhIcstmgAAwPGc4ubbhIQEhYeHS5Lmzp2rZs2aadiwYdq0aZMaN27s4OoAAMD1nGIGwsPDw/YyreXLl+uJJ56QJBUqVOimt3gCAIDbzylmIOrWravevXurTp06+uWXXzR79mxJ0p49e1SqVCkHVwcAAK7nFDMQH374odzc3DRnzhxNmDBBJUuWlCR99913atiwoYOrAwAA1+M2TgC3HbdxAs7L6R8kdb2MjAzNnz9fO3fulCTde++9evLJJ+Xq6urgygAAwPWcIkDs27dPjRs31tGjRxUWFibpymu6S5curcWLF6tcuXIOrhAAAFzLKdZA9OjRQ+XKldOff/6pTZs2adOmTTp8+LBCQkLUo0cPR5cHAACu4xQzEKtXr9aGDRts78aQpMKFC+u9995TnTp1HFgZAADIjlPMQFitVl24cCFLe1JSkjz+/1XOAADAeThFgGjatKk6d+6sjRs3yjAMGYahDRs2qGvXrnryyScdXR4AALiOUwSI2NhYhYaGqnbt2vL09JSnp6fq1Kmj0NBQjRs3ztHlAQCA6zh0DURmZqbef/99LViwQKmpqWrRooWio6NlsVhUqVIlhYaGOrI8AACQA4cGiKFDh2rQoEGKjIyUl5eXlixZIn9/f3322WeOLAsAANyEQy9hzJgxQx9//LF++OEHzZ8/XwsXLlRcXJwyMzMdWRYAALgJhwaIw4cP272uOzIyUhaLRceOHXNgVQAA4GYcGiDS09Pl6elp1+bu7q60tDQHVQQAAHLDoWsgDMNQTEyMrFarre3y5cvq2rWrfHx8bG3z5s1zRHkAACAHDg0Q0dHRWdrat2/vgEoAAIAZDg0QU6dOdeThAQDALXKKB0kBAIA7CwECAACYRoAAAACmESAAAIBpBAgAAGAaAQIAAJhGgAAAAKYRIAAAgGkECAAAYBoBAgAAmEaAAAAAphEgAACAaQQIAABgGgECAACYRoAAAACmESAAAIBpBAgAAGAaAQIAAJhGgAAAAKYRIAAAgGkECAAAYBoBAgAAmEaAAAAAphEgAACAaQQIAABgGgECAACYRoAAAACmESAAAIBpBAgAAGAaAQIAAJhGgAAAAKYRIAAAgGkECAAAYBoBAgAAmEaAAAAAphEgAACAaQQIAABgGgECAACYRoAAAACmESAAAIBpBAgAAGAaAQIAAJhGgAAAAKYRIAAAgGkECAAAYBoBAgAAmEaAAAAAphEgAACAaRbDMAxHFwHcSEpKioYPH66BAwfKarU6uhwA1+D7efciQMDpnT9/Xv7+/jp37pz8/PwcXQ6Aa/D9vHtxCQMAAJhGgAAAAKYRIAAAgGkECDg9q9Wqt99+mwVagBPi+3n3YhElAAAwjRkIAABgGgECAACYRoAAAACmESDwrxMcHKyxY8c6ugzgX23VqlWyWCw6e/bsDfvxffz3IkDAlJiYGFksFr333nt27fPnz5fFYrmttUybNk0FCxbM0v7rr7+qc+fOt7UWwFld/c5aLBZ5eHgoNDRUQ4YMUXp6+j8at3bt2kpMTJS/v78kvo93IwIETPP09NSIESN05swZR5eSraJFi8rb29vRZQBOo2HDhkpMTNTevXvVp08fDRo0SO+///4/GtPDw0PFihW76T8c+D7+exEgYFpkZKSKFSum4cOH59hn7dq1euSRR+Tl5aXSpUurR48eunjxom17YmKimjRpIi8vL4WEhGjWrFlZpjpHjx6typUry8fHR6VLl9bLL7+spKQkSVemTzt06KBz587Z/nU1aNAgSfZTpm3bttWzzz5rV1taWpqKFCmiGTNmSJIyMzM1fPhwhYSEyMvLS1WrVtWcOXPy4JMCnIPValWxYsUUFBSkbt26KTIyUgsWLNCZM2f0wgsvKCAgQN7e3mrUqJH27t1r2+/QoUNq1qyZAgIC5OPjo3vvvVdLliyRZH8Jg+/j3YkAAdNcXV01bNgwjR8/XkeOHMmyff/+/WrYsKGeeuopbd26VbNnz9batWv1yiuv2Pq88MILOnbsmFatWqW5c+dq0qRJOnHihN04Li4uio2N1fbt2zV9+nStWLFCr7/+uqQr06djx46Vn5+fEhMTlZiYqL59+2appV27dlq4cKEteEjSDz/8oOTkZLVs2VKSNHz4cM2YMUMTJ07U9u3b1atXL7Vv316rV6/Ok88LcDZeXl5KTU1VTEyMfvvtNy1YsEDr16+XYRhq3Lix0tLSJEndu3dXSkqK1qxZo23btmnEiBHy9fXNMh7fx7uUAZgQHR1tNG/e3DAMw3j44YeNF1980TAMw/jmm2+Mq/87dezY0ejcubPdfj/99JPh4uJiXLp0ydi5c6chyfj1119t2/fu3WtIMsaMGZPjsb/++mujcOHCtp+nTp1q+Pv7Z+kXFBRkGyctLc0oUqSIMWPGDNv25557znj22WcNwzCMy5cvG97e3sa6devsxujYsaPx3HPP3fjDAO4A135nMzMzjWXLlhlWq9Vo0aKFIcn4+eefbX1PnTpleHl5GV999ZVhGIZRuXJlY9CgQdmOu3LlSkOScebMGcMw+D7ejdwcml5wRxsxYoQeffTRLP/S2LJli7Zu3aq4uDhbm2EYyszMVEJCgvbs2SM3NzdVr17dtj00NFQBAQF24yxfvlzDhw/Xrl27dP78eaWnp+vy5ctKTk7O9TVVNzc3tW7dWnFxcXr++ed18eJFffvtt/ryyy8lSfv27VNycrIef/xxu/1SU1N1//33m/o8AGe1aNEi+fr6Ki0tTZmZmWrbtq1atWqlRYsWqWbNmrZ+hQsXVlhYmHbu3ClJ6tGjh7p166alS5cqMjJSTz31lKpUqXLLdfB9/HchQOCW1atXT1FRURo4cKBiYmJs7UlJSerSpYt69OiRZZ8yZcpoz549Nx374MGDatq0qbp166ahQ4eqUKFCWrt2rTp27KjU1FRTi7LatWuniIgInThxQsuWLZOXl5caNmxoq1WSFi9erJIlS9rtx7P98W/RoEEDTZgwQR4eHipRooTc3Ny0YMGCm+730ksvKSoqSosXL9bSpUs1fPhwjRo1Sq+++uot18L38d+DAIF/5L333lO1atUUFhZma6tevbp27Nih0NDQbPcJCwtTenq64uPjVaNGDUlX/uVx7V0dv//+uzIzMzVq1Ci5uFxZqvPVV1/ZjePh4aGMjIyb1li7dm2VLl1as2fP1nfffadnnnlG7u7ukqTw8HBZrVYdPnxYERER5k4euEP4+Phk+T5WqlRJ6enp2rhxo2rXri1JOn36tHbv3q3w8HBbv9KlS6tr167q2rWrBg4cqMmTJ2cbIPg+3n0IEPhHKleurHbt2ik2NtbW1r9/fz388MN65ZVX9NJLL8nHx0c7duzQsmXL9OGHH6pixYqKjIxU586dNWHCBLm7u6tPnz7y8vKy3RIWGhqqtLQ0jR8/Xs2aNdPPP/+siRMn2h07ODhYSUlJ+vHHH1W1alV5e3vnODPRtm1bTZw4UXv27NHKlStt7QUKFFDfvn3Vq1cvZWZmqm7dujp37px+/vln+fn5KTo6Oh8+NcDxypcvr+bNm6tTp0765JNPVKBAAQ0YMEAlS5ZU8+bNJUmvvfaaGjVqpAoVKujMmTNauXKlKlWqlO14fB/vQo5ehIE7y7ULsq5KSEgwPDw8jGv/d/rll1+Mxx9/3PD19TV8fHyMKlWqGEOHDrVtP3bsmNGoUSPDarUaQUFBxqxZs4zAwEBj4sSJtj6jR482ihcvbnh5eRlRUVHGjBkz7BZtGYZhdO3a1ShcuLAhyXj77bcNw7BftHXVjh07DElGUFCQkZmZabctMzPTGDt2rBEWFma4u7sbRYsWNaKioozVq1f/sw8LcALZfWev+vvvv43nn3/e8Pf3t33P9uzZY9v+yiuvGOXKlTOsVqtRtGhR4/nnnzdOnTplGEbWRZSGwffxbsPrvOEUjhw5otKlS2v58uV67LHHHF0OAOAmCBBwiBUrVigpKUmVK1dWYmKiXn/9dR09elR79uyxXQ8FADgv1kDAIdLS0vTGG2/owIEDKlCggGrXrq24uDjCAwDcIZiBAAAApvEoawAAYBoBAgAAmEaAAAAAphEgAACAaQQIAABgGgECQL6JiYlRixYtbD/Xr19fr7322m2vY9WqVbJYLDp79uxtPzbwb0WAAO5CMTExslgsslgs8vDwUGhoqIYMGaL09PR8Pe68efP0zjvv5Kovf+kDzo0HSQF3qYYNG2rq1KlKSUnRkiVL1L17d7m7u2vgwIF2/VJTU+Xh4ZEnxyxUqFCejAPA8ZiBAO5SVqtVxYoVU1BQkLp166bIyEgtWLDAdtlh6NChKlGihO1V7X/++adat26tggULqlChQmrevLkOHjxoGy8jI0O9e/dWwYIFVbhwYb3++uu6/jl111/CSElJUf/+/VW6dGlZrVaFhoZqypQpOnjwoBo0aCBJCggIkMViUUxMjCQpMzNTw4cPV0hIiLy8vFS1alXNmTPH7jhLlixRhQoV5OXlpQYNGtjVCSBvECAASJK8vLyUmpoqSfrxxx+1e/duLVu2TIsWLVJaWpqioqJUoEAB/fTTT/r555/l6+urhg0b2vYZNWqUpk2bps8++0xr167V33//rW+++eaGx3zhhRf0xRdfKDY2Vjt37tQnn3wiX19flS5dWnPnzpUk7d69W4mJiRo3bpwkafjw4ZoxY4YmTpyo7du3q1evXmrfvr1Wr14t6UrQadWqlZo1a6bNmzfrpZde0oABA/LrYwPuXg58EygAB7n2Fc+ZmZnGsmXLDKvVavTt29eIjo427rnnHiMlJcXWf+bMmUZYWJjdq5dTUlIMLy8v44cffjAMwzCKFy9ujBw50rY9LS3NKFWqlN2rpCMiIoyePXsahmEYu3fvNiQZy5Yty7bG7F4XffnyZcPb29tYt26dXd+OHTsazz33nGEYhjFw4EAjPDzcbnv//v2zjAXgn2ENBHCXWrRokXx9fZWWlqbMzEy1bdtWgwYNUvfu3VW5cmW7dQ9btmzRvn37VKBAAbsxLl++rP379+vcuXNKTExUzZo1bdvc3Nz0wAMPZLmMcdXmzZvl6uqqiIiIXNe8b98+JScn6/HHH7drT01N1f333y9J2rlzp10dklSrVq1cHwNA7hAggLtUgwYNNGHCBHl4eKhEiRJyc/vfHwc+Pj52fZOSklSjRg3FxcVlGado0aK3dHwvLy/T+yQlJUmSFi9erJIlS9pts1qtt1QHgFtDgADuUj4+PgoNDc1V3+rVq2v27NkKDAyUn59ftn2KFy+ujRs3ql69epKk9PR0/f7776pevXq2/StXrqzMzEytXr1akZGRWbZfnQHJyMiwtYWHh8tqterw4cM5zlxUqlRJCxYssGvbsGHDzU8SgCksogRwU+3atVORIkXUvHlz/fTTT0pISNCqVavUo0cPHTlyRJLUs2dPvffee5o/f7527dqll19++YbPcAgODlZ0dLRefPFFzZ8/3zbmV199JUkKCgqSxWLRokWLdPLkSSUlJalAgQLq27evevXqpenTp2v//v3atGmTxo8fr+nTp0uSunbtqr1796pfv37avXu3Zs2apWnTpuX3RwTcdQgQAG7K29tba9asUZkyZdSqVStVqlRJHTt21OXLl20zEn369NHzzz+v6Oho1apVSwUKFFDLli1vOO6ECRP09NNP6+WXX1bFihXVqVMnXbx4UZJUsmRJDR48WAMGDNA999yjV155RZL0zjvv6M0339Tw4cNVqVIlNWzYUIsXL1ZISIgkqUyZMpo7d67mz5+vqlWrauLEiRo2bFg+fjrA3cli5LTCCQAAIAfMQAAAANMIEAAAwDQCBAAAMI0AAQAATCNAAAAA0wgQAADANAIEAAAwjQABAABMI0AAAADTCBAAAMA0AgQAADDt/wAOD4az962KfgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load the fine-tuned model from hugging face"
      ],
      "metadata": {
        "id": "5w9b9a-2DZND"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile inference.py\n",
        "import torch\n",
        "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n",
        "\n",
        "model_name = 'luluw/distilbert-base-uncased-finetuned-sentiment'\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)\n",
        "model = DistilBertForSequenceClassification.from_pretrained(model_name)\n",
        "\n",
        "# Example usage\n",
        "text = \"I love this product!\"\n",
        "inputs = tokenizer(text, return_tensors='pt')\n",
        "outputs = model(**inputs)\n",
        "predictions = torch.argmax(outputs.logits, dim=-1)\n",
        "\n",
        "print(text)\n",
        "print(\"positive\" if predictions==1 else \"negative\")"
      ],
      "metadata": {
        "id": "SvnM8gDEDaDY"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 inference.py"
      ],
      "metadata": {
        "id": "IncD5nzoERpK"
      },
      "execution_count": 55,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "7904667e67424983aeacfdf6fceab2a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5e12365361ab4641a7ba003c86d77586",
              "IPY_MODEL_5db308d5b97f44c48fbedbbf0dbbf97e",
              "IPY_MODEL_8496b3a8f89b420bbd64ba2f7e6fa6f5"
            ],
            "layout": "IPY_MODEL_4c01bea3d08d4d7b956752b43697c996"
          }
        },
        "5e12365361ab4641a7ba003c86d77586": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_68b672b53c8a4a1097f47c2a850ce1a0",
            "placeholder": "​",
            "style": "IPY_MODEL_12f37aede6d249dd817b16db3b12b099",
            "value": "Preprocessing: 100%"
          }
        },
        "5db308d5b97f44c48fbedbbf0dbbf97e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f7794110d28344d8b713c63aa5ec0e21",
            "max": 20000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6090c48a151a42d5a4daa26662177e7c",
            "value": 20000
          }
        },
        "8496b3a8f89b420bbd64ba2f7e6fa6f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fc46c0fc11814ef9bb9045d2dad97234",
            "placeholder": "​",
            "style": "IPY_MODEL_79887438173e4ad4a55a46e735ec569c",
            "value": " 20000/20000 [00:05&lt;00:00, 4445.08it/s]"
          }
        },
        "4c01bea3d08d4d7b956752b43697c996": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68b672b53c8a4a1097f47c2a850ce1a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "12f37aede6d249dd817b16db3b12b099": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f7794110d28344d8b713c63aa5ec0e21": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6090c48a151a42d5a4daa26662177e7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fc46c0fc11814ef9bb9045d2dad97234": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79887438173e4ad4a55a46e735ec569c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "923e1b7fbd8b49bdabcd4f00f714be14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9f025708b4034c879c0a4249378fadc3",
              "IPY_MODEL_f03f81ec0bc44602be760ff2f548a5fd",
              "IPY_MODEL_038710f43d7a48dc8660458c680b35fb",
              "IPY_MODEL_368cc160a1bb43b7b11f7991b2d55d5a"
            ],
            "layout": "IPY_MODEL_b6caf3716cd4411cb727f34683958bf6"
          }
        },
        "b809d01b8cd344d98f2273b3ae8f7ffb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2480db893d3b43379c2aff68f421eb07",
            "placeholder": "​",
            "style": "IPY_MODEL_e8fd4b2f11b749ca9ef44d94353e13f0",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "6360e06ae7e74b89ad45dd1bb4413342": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_3bc4f28421f8430d8640f292e94d8c64",
            "placeholder": "​",
            "style": "IPY_MODEL_96edc1fed27340c692bf75fd4fc9c5d2",
            "value": ""
          }
        },
        "1421f131fe17439898b48f87cfa26b52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_ee83043d95cc4bee83a14456fbd7c01c",
            "style": "IPY_MODEL_ed42110cf0cf47f3aff0853d0452e7d8",
            "value": true
          }
        },
        "730e7bfb278a433bac4fedce9538cf0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_a5c25ef7145d4a819569b30c10b3db22",
            "style": "IPY_MODEL_be723d54691348868cff55a8ba046eab",
            "tooltip": ""
          }
        },
        "0cb660d95a9b46ed9ea2689b3b828508": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3242d629683242bfbc5777a2b904dd70",
            "placeholder": "​",
            "style": "IPY_MODEL_657bc2cc2cb744c4a1f72488b58d53b0",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "b6caf3716cd4411cb727f34683958bf6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "2480db893d3b43379c2aff68f421eb07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e8fd4b2f11b749ca9ef44d94353e13f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3bc4f28421f8430d8640f292e94d8c64": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96edc1fed27340c692bf75fd4fc9c5d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ee83043d95cc4bee83a14456fbd7c01c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed42110cf0cf47f3aff0853d0452e7d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a5c25ef7145d4a819569b30c10b3db22": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be723d54691348868cff55a8ba046eab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "3242d629683242bfbc5777a2b904dd70": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "657bc2cc2cb744c4a1f72488b58d53b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "35e38db3a6e74956ae22f6e6b3d594a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e800a5dc461b465c98814786c7b9d42b",
            "placeholder": "​",
            "style": "IPY_MODEL_90cea132a5114445922194fa5aea05d1",
            "value": "Connecting..."
          }
        },
        "e800a5dc461b465c98814786c7b9d42b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90cea132a5114445922194fa5aea05d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9f025708b4034c879c0a4249378fadc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_94bc6056062746bc8b1cfbfa3292a241",
            "placeholder": "​",
            "style": "IPY_MODEL_6fe80fdd065148a2b8f6eebf0862123a",
            "value": "Token is valid (permission: write)."
          }
        },
        "f03f81ec0bc44602be760ff2f548a5fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_680bcce08dd645f9badcfd5a9f53dc4f",
            "placeholder": "​",
            "style": "IPY_MODEL_1d5d3e06619c4084b41ab120e8476c6d",
            "value": "Your token has been saved in your configured git credential helpers (store)."
          }
        },
        "038710f43d7a48dc8660458c680b35fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f701050853814fff9c33373ab92deb43",
            "placeholder": "​",
            "style": "IPY_MODEL_2a4847e7a62f46b0b2afd22f9c1a0a4c",
            "value": "Your token has been saved to /root/.cache/huggingface/token"
          }
        },
        "368cc160a1bb43b7b11f7991b2d55d5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7b0317bfb8d742a09a3aa2c0ebb9773c",
            "placeholder": "​",
            "style": "IPY_MODEL_b848fa1f848348079650e1897429c3fa",
            "value": "Login successful"
          }
        },
        "94bc6056062746bc8b1cfbfa3292a241": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6fe80fdd065148a2b8f6eebf0862123a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "680bcce08dd645f9badcfd5a9f53dc4f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d5d3e06619c4084b41ab120e8476c6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f701050853814fff9c33373ab92deb43": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a4847e7a62f46b0b2afd22f9c1a0a4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7b0317bfb8d742a09a3aa2c0ebb9773c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b848fa1f848348079650e1897429c3fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "651b42c5bcb446e58ee60d07411982cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e2c20c8499e1439c83e12333b6876280",
              "IPY_MODEL_47d938799f6149bca0349e65366cad51",
              "IPY_MODEL_24b53ee1f6b44eb996574e5c8d9231b8"
            ],
            "layout": "IPY_MODEL_3f1d4ce564974fa5be80e15978afb915"
          }
        },
        "e2c20c8499e1439c83e12333b6876280": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_095ce34a8efb4ebab6df1faf3ab9257a",
            "placeholder": "​",
            "style": "IPY_MODEL_7f556609234146d08c78c0bbea32ec93",
            "value": "100%"
          }
        },
        "47d938799f6149bca0349e65366cad51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a71b7b35ac20470aa715857fe4eac0d3",
            "max": 4000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4cb94bfc2fc94e8b8787f41612b7f749",
            "value": 4000
          }
        },
        "24b53ee1f6b44eb996574e5c8d9231b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_54f7632a7d3047ce9b6e9333ac871248",
            "placeholder": "​",
            "style": "IPY_MODEL_198d620b60f14cd2aa5cfb0114dec311",
            "value": " 4000/4000 [00:56&lt;00:00, 96.18it/s]"
          }
        },
        "3f1d4ce564974fa5be80e15978afb915": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "095ce34a8efb4ebab6df1faf3ab9257a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7f556609234146d08c78c0bbea32ec93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a71b7b35ac20470aa715857fe4eac0d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4cb94bfc2fc94e8b8787f41612b7f749": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "54f7632a7d3047ce9b6e9333ac871248": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "198d620b60f14cd2aa5cfb0114dec311": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}