T2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "93b39b25-e5cf-46f6-9b53-20672b4f8748",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:15:49.667323Z",
     "iopub.status.busy": "2024-10-18T07:15:49.666933Z",
     "iopub.status.idle": "2024-10-18T07:15:49.938467Z",
     "shell.execute_reply": "2024-10-18T07:15:49.937755Z",
     "shell.execute_reply.started": "2024-10-18T07:15:49.667301Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85af8cc053ba4b59ad2638232acdcfd5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bba6f5e2-db4e-42ae-8908-83ac30e51965",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:15:56.525533Z",
     "iopub.status.busy": "2024-10-18T07:15:56.525109Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "transformers 4.38.2 requires tokenizers<0.19,>=0.14, but you have tokenizers 0.20.1 which is incompatible.\n",
      "unbabel-comet 2.2.2 requires protobuf<5.0.0,>=4.24.4, but you have protobuf 3.20.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "unbabel-comet 2.2.2 requires protobuf<5.0.0,>=4.24.4, but you have protobuf 3.20.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%pip install --upgrade torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 -q\n",
    "%pip install -U tokenizers -q\n",
    "%pip install peft transformers protobuf==3.20 bitsandbytes sentencepiece sacrebleu ipython datasets evaluate deepspeed einops wandb zstandard accelerate jsonlines trl tokenizers -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bdbfea81-861c-4e73-9bce-fd7e1d5939ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:49:48.482948Z",
     "iopub.status.busy": "2024-10-18T06:49:48.482471Z",
     "iopub.status.idle": "2024-10-18T07:16:22.494293Z",
     "shell.execute_reply": "2024-10-18T07:16:22.493397Z",
     "shell.execute_reply.started": "2024-10-18T07:16:16.041816Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/.local/lib/python3.10/site-packages/transformers/utils/hub.py:124: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from peft import PeftModel, PeftConfig\n",
    "\n",
    "from comet import download_model, load_from_checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9e1dfd21-ba50-4c4e-9904-987d67b40f86",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:58:19.462610Z",
     "iopub.status.busy": "2024-10-18T06:58:19.462236Z",
     "iopub.status.idle": "2024-10-18T06:58:19.490059Z",
     "shell.execute_reply": "2024-10-18T06:58:19.489383Z",
     "shell.execute_reply.started": "2024-10-18T06:58:19.462589Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('data/translate_cpo_kiwi.json', 'r') as f:\n",
    "    data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c5ca599e-cf93-409f-b146-aa6f9e8615af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:58:19.553219Z",
     "iopub.status.busy": "2024-10-18T06:58:19.552806Z",
     "iopub.status.idle": "2024-10-18T06:58:19.569309Z",
     "shell.execute_reply": "2024-10-18T06:58:19.568614Z",
     "shell.execute_reply.started": "2024-10-18T06:58:19.553201Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def adequacy(en_tr, en_ref):\n",
    "    return len(en_tr) / len(en_ref)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7c4fc8fa-b3f7-459b-859c-554dbefc279d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:58:19.692821Z",
     "iopub.status.busy": "2024-10-18T06:58:19.692355Z",
     "iopub.status.idle": "2024-10-18T06:58:19.710454Z",
     "shell.execute_reply": "2024-10-18T06:58:19.709814Z",
     "shell.execute_reply.started": "2024-10-18T06:58:19.692803Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "adequacy_score = [adequacy(item['mt'], item['ref']) for item in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d557e0c6-ae48-4292-9fbb-6a276bea1df2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:58:19.833845Z",
     "iopub.status.busy": "2024-10-18T06:58:19.833343Z",
     "iopub.status.idle": "2024-10-18T06:58:20.028434Z",
     "shell.execute_reply": "2024-10-18T06:58:20.027689Z",
     "shell.execute_reply.started": "2024-10-18T06:58:19.833818Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Отношение длины перевода на EN к референсу EN')"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGzCAYAAAAFROyYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8vElEQVR4nO3deVxU9eL/8fcgMqAChoqA4UZumVt2Jcs1NcQty3stt6zMpSxv+miR0pSWi6a3vJWVdk27uWaardd9rdDrei1v4UqWCm4BigkCn98f/Ziv4wzIKCMHfD0fj/N4MJ/zOed8zmfOzLz5nHNmbMYYIwAAAAvxKekGAAAAXIqAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAchl79uzRwIEDVaNGDdntdkVERGjAgAHas2dPSTcNQBmQnJwsm81W4DRp0iRH3Q4dOshms6lnz54Frmfq1KnXsvle8cknn8hms2n9+vUl3ZRSIf+4cDc1bNjQUW/OnDmy2Wzy9/fXkSNH3K7nlltuuZZNL5RvSTfAypYuXap+/fopJCREQ4YMUZ06dZScnKxZs2bpk08+0cKFC3XvvfeWdDMBlAH9+vVTt27dXMpbtGjhUvbll19q+/btatmy5bVoGkqBG2+8UQkJCS7lwcHBLmVZWVmaNGmS3nrrrWvRtCtGQCnAgQMHNGjQINWtW1cbN25UtWrVHPP++te/qm3btho0aJB2796tunXrlmBLAZQFt956qwYOHHjZejVr1tSZM2cUHx+vzz///Bq0DKVBcHBwkY4fSWrevLnef/99xcXFKSIiwsstu3Kc4inAlClTdO7cOc2cOdMpnEhS1apVNWPGDGVmZuq1116TJE2cOLHQYdqLhysLGkabOnWqbDabkpOTncrfeecdNW7c2HGKaeTIkUpLS3Oq426d7ta3f/9+2Ww2vf322051f/rpJ/35z39WSEiI/P39ddttt7m8+eUPD27bts2p/OTJk7LZbJo4caKjLL8/Lnb27FmFhYW5HbrdsmWLunbtquDgYFWoUEHt27fXt99+69JHBSmo/zt06OBSt7Ah9UvrzJkzx2nZkSNHymaz6aGHHrqqfjl58mSB+1K7dm2n9btzudMCl+53VlaWJkyYoJtuukl2u12RkZF69tlnlZWV5VTPZrPpiSee0Lx589SgQQP5+/urZcuW2rhxo0sbjhw5okceeUTVq1eX3W5X48aN9cEHHzjVWb9+vVO77Ha76tevr4SEBF36Q+o7d+5UbGysgoKCVKlSJXXq1EmbN292u/8PPfSQ2/2+tN8+++wzde/eXREREbLb7YqKitLLL7+s3NzcQvtXKvi52rZtm8uxsXv3bj300EOqW7eu/P39FRYWpkceeUSnTp267HY8FRgYqNGjR+uLL77Qjh07PF7e3bF95swZtWzZUnXq1NGxY8cKXb527drq0aOHVq5cqebNm8vf318333yzli5d6lI3LS1NTz31lCIjI2W323XTTTdp8uTJysvLc6p38OBBtW/fXgEBAbr77rsd/XbkyBG1adNGAQEB6tGjh06cOOGyjblz56ply5YKCAhQSEiIHnjgAf3yyy9OdfLfH7dv36477rhDAQEBqlOnjt577z2X9Xn6WrlUjx49VLt2baeyvLw8/eMf/1CTJk3k7++vatWqqWvXro73jPbt26tZs2Zueltq0KCBYmJi3M67Us8//7xyc3OdTh9aESMoBfjiiy9Uu3ZttW3b1u38du3aqXbt2vrqq68kSffdd59uuukmx/zRo0erUaNGGjZsmKOsUaNGHrdj4sSJio+PV+fOnfXYY48pKSlJ7777rrZu3apvv/1W5cuX93idl9qzZ4/uvPNO1ahRQ2PHjlXFihX18ccfq3fv3lqyZEmxncb6+9//rtTUVJfytWvXKjY2Vi1bttSECRPk4+Oj2bNn66677tKmTZvUqlWrIm/j3XffVaVKlSRJcXFxhdYdNmyY4/ldunSpPv3000Lr79+/X++//36R23ItuDstcOl+5+XlqVevXvrmm280bNgwNWrUSN9//73eeOMN7d27V8uWLXOqv2HDBi1atEijRo2S3W7XO++8o65du+o///mPIwSnpqbq9ttvd7xJV6tWTf/+9781ZMgQZWRk6KmnnnJa5/PPP69GjRrp999/16JFi/T8888rNDRUQ4YMkfTHMdi2bVsFBQXp2WefVfny5TVjxgx16NBBGzZsUHR0tMu+2+12/fOf/3Q8fvTRR13qzJkzR5UqVdKYMWNUqVIlrV27Vi+++KIyMjI0ZcqUIvfz5axatUoHDx7Uww8/rLCwMO3Zs0czZ87Unj17tHnzZpew7s65c+fcBtfKlSvL19f5rfqvf/2r3njjDU2cOPGqR1EuXLigPn366PDhw/r2228VHh5+2WX27dun+++/XyNGjNDgwYM1e/Zs/eUvf9Hy5cvVpUsXx/60b99eR44c0fDhw1WzZk199913iouL07FjxzRt2jRJfxyfPXr00LFjx/Tiiy8qNTVV48ePlyQ9/fTTGjJkiDp27KgpU6Zo4MCBWrFihaMdr776qsaPH6++ffvq0Ucf1YkTJ/TWW2+pXbt22rlzpypXruyo+9tvv6lbt27q27ev+vXrp48//liPPfaY/Pz89Mgjjzja4slrpaiGDBmiOXPmKDY2Vo8++qhycnK0adMmbd68WbfddpsGDRqkoUOH6ocffnD6R3Pr1q3au3evxo0bd9lt5Obmuj1+AgICVLFiRaeyOnXq6MEHH9T777+vsWPHWncUxcBFWlqakWTuueeeQuv16tXLSDIZGRku82rVqmUGDx7sdrn27dubxo0bu5RPmTLFSDKHDh0yxhhz/Phx4+fnZ+6++26Tm5vrqPf2228bSeaDDz4odJ2Xrs8YY/bt22ckmbfeestR1qlTJ9OkSRNz/vx5R1leXp654447TL169Rxls2fPNpLM1q1bnbZz4sQJI8lMmDDBUTZhwgRz8eF1/PhxExgYaGJjY40ks27dOsd26tWrZ2JiYkxeXp6j/rlz50ydOnVMly5d3HWhi+eff95IMidPnnSUNW7c2LRv396lbn4ffPjhhwW299ChQ0aSmT17tqOsb9++5pZbbjGRkZFOz+2V9MuJEycK3JfCjp1L2zdlyhSXeZfu90cffWR8fHzMpk2bnOq99957RpL59ttvHWWSjCSzbds2R9nPP/9s/P39zb333usoGzJkiAkPD3fqb2OMeeCBB0xwcLA5d+6cMcaYdevWOT3fxhhz/vx54+PjYx5//HFHWe/evY2fn585cOCAo+zo0aMmMDDQtGvXzmUf+/fvbypVquRUVrFiRZd+y2/HxYYPH24qVKjgdLy7U9BztXXrVpdjw912FixYYCSZjRs3Frqd/OeyoCkxMdFR9+LXeXx8vJFktm/f7rQed8eEu+3Nnj3b5OXlmQEDBpgKFSqYLVu2FLpcvlq1ahlJZsmSJY6y9PR0Ex4eblq0aOEoe/nll03FihXN3r17nZYfO3asKVeunDl8+LAxxpilS5caSWbFihWOOk8//bSRZCZPnuwomzFjhpFkduzYYYwxJjk52ZQrV868+uqrTuv//vvvja+vr1N5+/btjSTz97//3VGWlZVlmjdvbkJDQ012drYxxvPXysiRI136p3v37qZWrVqOx2vXrjWSzKhRo1zq5r/npaWlGX9/f/Pcc885zR81apSpWLGiOXv2rMuyF8vfP3fT8OHDHfUufq86cOCA8fX1dWpXQZ9NJYVTPG6cOXNG0h9DqYXJn5+RkeHxNvLT7sXTuXPnnOqsXr1a2dnZeuqpp+Tj839P1dChQxUUFOQYvbkap0+f1tq1a9W3b1+dOXPG0ZZTp04pJiZG+/btc7naOz093andp0+fvux2Xn75ZQUHB2vUqFFO5bt27dK+ffvUv39/nTp1yrHOzMxMderUSRs3bnQZDnbn/PnzkiR/f//L1s3Ozpb0x3/gRbV9+3YtXrxYCQkJTs/FxTzpl9OnTzv209sWL16sRo0aqWHDhk7tu+uuuyRJ69atc6rfunVrp4sva9asqXvuuUcrVqxQbm6ujDFasmSJevbsKWOM0zpjYmKUnp7ucuohv28OHz6s1157TXl5eY7t5+bmauXKlerdu7fT9Vzh4eHq37+/vvnmG5fX2Pnz54v0XAcEBDj+zj++27Ztq3Pnzumnn34qUv/lP1f5U3p6eqHbOX/+vE6ePKnbb79dkop8GmbYsGFatWqVy3TzzTe7rf/Xv/5VN9xwg+Lj44u0fneeeeYZzZs3Tx9//LFHI5URERFOI6tBQUF68MEHtXPnTqWkpEj647hr27atbrjhBqf+69y5s3Jzcx2nDdesWaPAwEDHyIskx4jZxW267777JP0x4ir9MeqZl5envn37Oq0/LCxM9erVczmufX19NXz4cMdjPz8/DR8+XMePH9f27dsdbfbktZL/XF88XbhwwanOkiVLZLPZNGHCBJd+zB9ZCw4O1j333KMFCxY4Tn3m5uZq0aJF6t27t8sIiDu1a9d2e/xcOpqZr27duho0aJBmzpx52dN6JYVTPG7kB4/8oFKQogYZd3766SeXa1su9fPPP0v64xzkxfz8/FS3bl3H/Kuxf/9+GWM0fvx4x7DqpY4fP64aNWo4Hnfu3NmjbRw6dEgzZszQu+++6/Khsm/fPknS4MGDC1w+PT1dN9xwQ6HbOHnypMqXL68KFSpctj351+/knwoqirFjx6pt27bq0aOH2/POkmf9cvFzGhoaqqFDhyo+Pl7lypUr8jqKat++ffrxxx8LPN6OHz/u9LhevXouderXr69z587pxIkT8vHxUVpammbOnKmZM2cWaZ29e/d2/O3j46Nx48apT58+kqQTJ07o3LlzLse59Mdp0by8PP3yyy9q3Lixo/zkyZNu70641J49ezRu3DitXbvWJeS4CxruuGvXpU6fPq34+HgtXLjQZd+Lup169ep5dAwFBwfrqaee0oQJE7Rz587LvkYuNWPGDMc1Pr/99ptHy950000up63q168v6Y9rXMLCwrRv3z7t3r37ssfdL7/8ovDw8MueBqtatar8/Pwc15fs27dPxhi3x6skl9PfERERLh/0F7f59ttv9/i1MmvWLM2aNculXq1atRx/HzhwQBEREQoJCSl0/x588EEtWrRImzZtUrt27bR69WqlpqZq0KBBhS6Xr2LFih6/N48bN04fffSRJk2apH/84x8eLXstEFDcCA4OVnh4uHbv3l1ovd27d6tGjRoKCgryeBu1a9d2uZ5h8eLFBb7he0v+6MTTTz9d4IVYF19bI0nTp093vLClP0aQ8j9s3HnhhRdUr149DR48WJs2bXK7/SlTpqh58+Zuly9KkEhOTlbNmjWLdK4//z+8sLCwy9aVpJUrV2r16tVKTEwstJ4n/bJkyRIFBQXp3Llz+vTTT/Xqq686rr8obnl5eWrSpIlef/11t/MjIyM9Xp8kDRw4sMBg2bRpU6fHU6dOVbNmzXThwgVt3bpVr7zyinx9fd3+V1kUycnJBX4w5UtLS1P79u0VFBSkl156SVFRUfL399eOHTv03HPPFWlkTvq/5yrf3r17NXLkSKc6ffv21XfffadnnnlGzZs3V6VKlZSXl6euXbsWeTtXIv9alPj4eMc1HUW1efNmvfrqq9q6datGjx6trl27qmrVqsXWtry8PHXp0qXAYzr/tZI/+llUv//+u2P9NptN//73v90Ge0/+Acnn6WvlnnvucfmHZdy4cY73GE/ExMSoevXqmjt3rtq1a6e5c+cqLCzM49Dhibp162rgwIGaOXOmxo4d67XtXCkCSgF69Oih999/X998843atGnjMn/Tpk1KTk52GjL0hLu0u2vXLqfH+Sk8KSnJaeg7Oztbhw4dKpYDN3+95cuXL/L6WrVqpdtuu83xuLA7Unbu3KmFCxdq2bJlbt9EoqKiJP0xRHyl+5OTk6P//ve/6tq1a5Hq/+9//5PNZivSf8bGGI0dO1b33nuvY8i+IJ70S7t27RwfBr169dK3336r5cuXeyWgREVF6b///a86depUpACXP6p1sb1796pChQqO/ywDAwOVm5tb5OesZcuWjjuLYmNjdeTIEU2ePFnjx49XtWrVVKFCBSUlJbks99NPP8nHx8fpg+HEiRM6fPiw+vXrV+g2169fr1OnTmnp0qVq166do/zQoUNFanO+i58rSU4XXkp/jD6sWbNG8fHxevHFFx3l7vqxuOWPokycOLHQUUh3HnnkET3//PM6evSobr75Zo0ePVofffRRkZbNH3m9+Hjau3evJDnuYImKitLZs2cve4yEh4crMTHRZX2XOnnypLKzsx0XdEZFRckYozp16jj9Y1CQo0ePKjMz02kUxV2bPXmt3HjjjS77N23aNKeAEhUVpRUrVuj06dOFjqKUK1dO/fv315w5czR58mQtW7ZMQ4cO9cqo6sXGjRunuXPnavLkyV7dzpXgGpQCPPPMMwoICNDw4cNdbhU8ffq0RowYoQoVKuiZZ57xWhs6d+4sPz8/vfnmm063ZM6aNUvp6enq3r37VW8jNDRUHTp00IwZM9yeh3R3W58nxo4dqzvvvFO9evVyO79ly5aKiorS1KlTdfbs2Sva/sqVK5Wenq577rnnsnVzcnK0ZMkStWrVqkj/YS1cuFC7d+92+wVIxcUYI2OM196I+vbtqyNHjri9A+n33393uQ4mMTHR6bqJX375RZ999pnuvvtulStXTuXKlVOfPn20ZMkS/fDDDy7rLMpz9vvvvysnJ0c5OTkqV66c7r77bn322WdOt8SnpqZq/vz5atOmjdMIxuLFiyXpss93fn9e/NrJzs7WO++8c9n2ecLddiR5PKJxpZ566ilVrlxZL730kkfL5d/BFhERocmTJ2vu3LlauXJlkZY9evSo011vGRkZ+te//qXmzZs7Rib79u2rxMREp7tu8qWlpSknJ0fSHwHwzJkzWrNmjWP+li1bJEn/+c9/HGX5d9Dkh8377rtP5cqVU3x8vEvfG2Nc3rdzcnI0Y8YMx+Ps7GzNmDFD1apVc1xz5elrpSj69OkjY4zba4UubfegQYP022+/afjw4Tp79myRv9fkakRFRWngwIGaMWPGFY38eBMjKAWoV6+ePvzwQw0YMEBNmjRx+SbZkydPasGCBY4RAG+oVq2a4uLiFB8fr65du6pXr15KSkrSO++8oz/96U8uB+/Zs2e1fPlyx+P8/0g3bNjguCDQXQiZPn262rRpoyZNmmjo0KGqW7euUlNTlZiYqF9//VX//e9/r3gfVq5cWej3mfj4+Oif//ynYmNj1bhxYz388MOqUaOGjhw5onXr1ikoKEhffPFFgcsvWrRITz/9tOx2u37//XfNnTvXMS89PV25ublatmyZevfurdWrV2v8+PHavXt3oeu8tP1Dhw4t0miLJ9auXet0imf//v0FXsx2tQYNGqSPP/5YI0aM0Lp163TnnXcqNzdXP/30kz7++GOtWLHCaeTnlltuUUxMjNNtxpKc3mAnTZqkdevWKTo6WkOHDtXNN9+s06dPa8eOHVq9erXLBcKrVq3Sr7/+6jjFM2/ePPXq1Ut+fn6SpFdeeUWrVq1SmzZt9Pjjj8vX11czZsxQVlaW47uGpD+O1XHjxqlatWo6cOCADhw44JiXk5OjgwcPatWqVerSpYvuuOMO3XDDDRo8eLBGjRolm82mjz76yOVD4WoFBQWpXbt2eu2113ThwgXVqFFDK1eu9HikZseOHU7Hb76oqCi1bt26wOWCg4P117/+9aoulh02bJjmz5+vESNG6IcffrjstVz169fXkCFDtHXrVlWvXl0ffPCBUlNTNXv2bEedZ555Rp9//rl69Oihhx56SC1btlRmZqa+//57ffLJJ0pOTlbVqlXVr18/vfLKK+rbt6+effZZpaSkaOHChZL+CHkZGRny8fHRlClT1Lp1a8dIXFRUlF555RXFxcUpOTlZvXv3VmBgoA4dOqRPP/1Uw4YN09NPP+1oT34QS05OVv369bVo0SLt2rVLM2fOdFyv4ulrpSg6duyoQYMG6c0339S+ffscp/02bdqkjh07Op0iatGihW655RbHxbq33nprkbeTnp7u9viRdNmg88ILL+ijjz5SUlKS07VeJe4a3zVU6uzevdv069fPhIeHm/Lly5uwsDDTr18/8/333xe6XHHcZpzv7bffNg0bNjTly5c31atXN4899pj57bffXNapQm5VvHS6+DZjY4w5cOCAefDBB01YWJgpX768qVGjhunRo4f55JNPHHWu5HbaS2/VdnfbqTHG7Ny509x3332mSpUqxm63m1q1apm+ffuaNWvWuO3DfPm3PBY25d/y9+STT5p27dqZ5cuXu6ynoNuMAwICzJEjR1y2ebW3GedPAQEB5uabbzZvvPFGget3x5PbjI0xJjs720yePNk0btzY2O12c8MNN5iWLVua+Ph4k56e7qin/3/r5Ny5c029evWM3W43LVq0cHm+jDEmNTXVjBw50kRGRjpeG506dTIzZ8501Ml/vvMnX19fU6tWLTNq1CiXY3jHjh0mJibGVKpUyVSoUMF07NjRfPfdd051inJsX7zv3377rbn99ttNQECAiYiIMM8++6xZsWKF22PwUp7cZvzrr7+ae++911SuXNkEBwebv/zlL+bo0aMuz787l7vN+OJjoaD3jt9++80EBwd7fJvxxZKSkoy/v78ZPXp0ocvXqlXLdO/e3axYscI0bdrU2O1207BhQ7N48WKXumfOnDFxcXHmpptuMn5+fqZq1armjjvuMFOnTnXc2pu/7TvvvNP4+/ubLl26OG4pnjt3rqO8a9eu5tixYy7bWLJkiWnTpo2pWLGiqVixomnYsKEZOXKkSUpKcum3bdu2mdatWxt/f39Tq1Yt8/bbb7usz9PXyqUuvc3YGGNycnLMlClTTMOGDY2fn5+pVq2aiY2NddwifrHXXnvNSDJ/+9vf3D8Bblzu/T9fQe9VxhgzePBgI8lStxkTUFCq1apVy+WN9mLr1q1zebNAwQp607WKywWL2bNnu/3uGxSf/IDiTYsXLy5SiCwqq32/R2GmTZtmbDab+fnnn0u6KSWOa1AAALAAY4xmzZql9u3bq2bNmiXdnBLHNSgo1e69995CrwOqXr06vzhdhgwYMEDVq1cvcH5UVJTTF34BpUFmZqY+//xzrVu3Tt9//70+++yzkm6SJRBQUKq98cYbhc5v1KjRZeug9CjoIsB8bdu2LfD3swCrOnHihPr376/KlSvr+eefL/Cux+uNzZhivqQdAADgKnENCgAAsBwCCgAAsJxSeQ1KXl6ejh49qsDAwCJ9HTEAACh5xhidOXNGERERBf4yfL5SGVCOHj3q8Q+cAQAAa/jll1904403FlqnVAaUwMBASX/s4JX8kjAAALj2MjIyFBkZ6fgcL0ypDCj5p3WCgoIIKAAAlDJFuTyDi2QBAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDl+JZ0AwB4V+2xX3lt3cmTuntt3QCub4ygAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAy/E4oGzcuFE9e/ZURESEbDabli1b5jTfZrO5naZMmeKoU7t2bZf5kyZNuuqdAQAAZYPHASUzM1PNmjXT9OnT3c4/duyY0/TBBx/IZrOpT58+TvVeeuklp3pPPvnkle0BAAAoczz+HpTY2FjFxsYWOD8sLMzp8WeffaaOHTuqbt26TuWBgYEudYHrlTe/qwQASiOvXoOSmpqqr776SkOGDHGZN2nSJFWpUkUtWrTQlClTlJOTU+B6srKylJGR4TQBAICyy6vfJPvhhx8qMDBQ9913n1P5qFGjdOuttyokJETfffed4uLidOzYMb3++utu15OQkKD4+HhvNhUAAFiIVwPKBx98oAEDBsjf39+pfMyYMY6/mzZtKj8/Pw0fPlwJCQmy2+0u64mLi3NaJiMjQ5GRkd5rOAAAKFFeCyibNm1SUlKSFi1adNm60dHRysnJUXJysho0aOAy3263uw0uAACgbPLaNSizZs1Sy5Yt1axZs8vW3bVrl3x8fBQaGuqt5gAAgFLE4xGUs2fPav/+/Y7Hhw4d0q5duxQSEqKaNWtK+uMUzOLFi/X3v//dZfnExERt2bJFHTt2VGBgoBITEzV69GgNHDhQN9xww1XsCgAAKCs8Dijbtm1Tx44dHY/zrw0ZPHiw5syZI0lauHChjDHq16+fy/J2u10LFy7UxIkTlZWVpTp16mj06NFO15gAAIDrm80YY0q6EZ7KyMhQcHCw0tPTFRQUVNLNAa5aaf0elORJ3Uu6CQBKEU8+v/ktHgAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDm+Jd0AoDSpPfarkm4CAFwXGEEBAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACW43FA2bhxo3r27KmIiAjZbDYtW7bMaf5DDz0km83mNHXt2tWpzunTpzVgwAAFBQWpcuXKGjJkiM6ePXtVOwIAAMoOjwNKZmammjVrpunTpxdYp2vXrjp27JhjWrBggdP8AQMGaM+ePVq1apW+/PJLbdy4UcOGDfO89QAAoEzy9XSB2NhYxcbGFlrHbrcrLCzM7bwff/xRy5cv19atW3XbbbdJkt566y1169ZNU6dOVUREhKdNAgAAZYxXrkFZv369QkND1aBBAz322GM6deqUY15iYqIqV67sCCeS1LlzZ/n4+GjLli1u15eVlaWMjAynCQAAlF3FHlC6du2qf/3rX1qzZo0mT56sDRs2KDY2Vrm5uZKklJQUhYaGOi3j6+urkJAQpaSkuF1nQkKCgoODHVNkZGRxNxsAAFiIx6d4LueBBx5w/N2kSRM1bdpUUVFRWr9+vTp16nRF64yLi9OYMWMcjzMyMggpAACUYV6/zbhu3bqqWrWq9u/fL0kKCwvT8ePHnerk5OTo9OnTBV63YrfbFRQU5DQBAICyy+sB5ddff9WpU6cUHh4uSWrdurXS0tK0fft2R521a9cqLy9P0dHR3m4OAAAoBTw+xXP27FnHaIgkHTp0SLt27VJISIhCQkIUHx+vPn36KCwsTAcOHNCzzz6rm266STExMZKkRo0aqWvXrho6dKjee+89XbhwQU888YQeeOAB7uABAACSrmAEZdu2bWrRooVatGghSRozZoxatGihF198UeXKldPu3bvVq1cv1a9fX0OGDFHLli21adMm2e12xzrmzZunhg0bqlOnTurWrZvatGmjmTNnFt9eAQCAUs3jEZQOHTrIGFPg/BUrVlx2HSEhIZo/f76nmwYAANcJfosHAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYjm9JNwBA6VV77FdeWW/ypO5eWS+A0oMRFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkeB5SNGzeqZ8+eioiIkM1m07JlyxzzLly4oOeee05NmjRRxYoVFRERoQcffFBHjx51Wkft2rVls9mcpkmTJl31zgAAgLLB44CSmZmpZs2aafr06S7zzp07px07dmj8+PHasWOHli5dqqSkJPXq1cul7ksvvaRjx445pieffPLK9gAAAJQ5vp4uEBsbq9jYWLfzgoODtWrVKqeyt99+W61atdLhw4dVs2ZNR3lgYKDCwsI83TwAALgOeP0alPT0dNlsNlWuXNmpfNKkSapSpYpatGihKVOmKCcnp8B1ZGVlKSMjw2kCAABll8cjKJ44f/68nnvuOfXr109BQUGO8lGjRunWW29VSEiIvvvuO8XFxenYsWN6/fXX3a4nISFB8fHx3mwqAACwEK8FlAsXLqhv374yxujdd991mjdmzBjH302bNpWfn5+GDx+uhIQE2e12l3XFxcU5LZORkaHIyEhvNR0AAJQwrwSU/HDy888/a+3atU6jJ+5ER0crJydHycnJatCggct8u93uNrgAAICyqdgDSn442bdvn9atW6cqVapcdpldu3bJx8dHoaGhxd0cAABQCnkcUM6ePav9+/c7Hh86dEi7du1SSEiIwsPD9ec//1k7duzQl19+qdzcXKWkpEiSQkJC5Ofnp8TERG3ZskUdO3ZUYGCgEhMTNXr0aA0cOFA33HBD8e0ZAAAotTwOKNu2bVPHjh0dj/OvDRk8eLAmTpyozz//XJLUvHlzp+XWrVunDh06yG63a+HChZo4caKysrJUp04djR492ukaEwAAcH3zOKB06NBBxpgC5xc2T5JuvfVWbd682dPNAgCA6wi/xQMAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACzH44CyceNG9ezZUxEREbLZbFq2bJnTfGOMXnzxRYWHhysgIECdO3fWvn37nOqcPn1aAwYMUFBQkCpXrqwhQ4bo7NmzV7UjAACg7PA4oGRmZqpZs2aaPn262/mvvfaa3nzzTb333nvasmWLKlasqJiYGJ0/f95RZ8CAAdqzZ49WrVqlL7/8Uhs3btSwYcOufC8AAECZ4uvpArGxsYqNjXU7zxijadOmady4cbrnnnskSf/6179UvXp1LVu2TA888IB+/PFHLV++XFu3btVtt90mSXrrrbfUrVs3TZ06VREREVexOwAAoCwo1mtQDh06pJSUFHXu3NlRFhwcrOjoaCUmJkqSEhMTVblyZUc4kaTOnTvLx8dHW7ZscbverKwsZWRkOE0AAKDsKtaAkpKSIkmqXr26U3n16tUd81JSUhQaGuo039fXVyEhIY46l0pISFBwcLBjioyMLM5mAwAAiykVd/HExcUpPT3dMf3yyy8l3SQAAOBFxRpQwsLCJEmpqalO5ampqY55YWFhOn78uNP8nJwcnT592lHnUna7XUFBQU4TAAAou4o1oNSpU0dhYWFas2aNoywjI0NbtmxR69atJUmtW7dWWlqatm/f7qizdu1a5eXlKTo6ujibAwAASimP7+I5e/as9u/f73h86NAh7dq1SyEhIapZs6aeeuopvfLKK6pXr57q1Kmj8ePHKyIiQr1795YkNWrUSF27dtXQoUP13nvv6cKFC3riiSf0wAMPcAcPAACQdAUBZdu2berYsaPj8ZgxYyRJgwcP1pw5c/Tss88qMzNTw4YNU1pamtq0aaPly5fL39/fscy8efP0xBNPqFOnTvLx8VGfPn305ptvFsPuAACAssBmjDEl3QhPZWRkKDg4WOnp6VyPgmuq9tivSroJ14XkSd1LugkAvMCTz+9ScRcPAAC4vhBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5fiWdAMA4FK1x37ltXUnT+rutXUDKD6MoAAAAMshoAAAAMshoAAAAMshoAAAAMshoAAAAMshoAAAAMshoAAAAMvhe1BQ5njzOzQAANcGIygAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByij2g1K5dWzabzWUaOXKkJKlDhw4u80aMGFHczQAAAKVYsX9R29atW5Wbm+t4/MMPP6hLly76y1/+4igbOnSoXnrpJcfjChUqFHczAABAKVbsAaVatWpOjydNmqSoqCi1b9/eUVahQgWFhYUV96YBAEAZ4dVrULKzszV37lw98sgjstlsjvJ58+apatWquuWWWxQXF6dz584Vup6srCxlZGQ4TQAAoOzy6m/xLFu2TGlpaXrooYccZf3791etWrUUERGh3bt367nnnlNSUpKWLl1a4HoSEhIUHx/vzaYCAAALsRljjLdWHhMTIz8/P33xxRcF1lm7dq06deqk/fv3Kyoqym2drKwsZWVlOR5nZGQoMjJS6enpCgoKKvZ2o3TjxwJRmORJ3Uu6CcB1KyMjQ8HBwUX6/PbaCMrPP/+s1atXFzoyIknR0dGSVGhAsdvtstvtxd5GAABgTV67BmX27NkKDQ1V9+6F/7eya9cuSVJ4eLi3mgIAAEoZr4yg5OXlafbs2Ro8eLB8ff9vEwcOHND8+fPVrVs3ValSRbt379bo0aPVrl07NW3a1BtNAQAApZBXAsrq1at1+PBhPfLII07lfn5+Wr16taZNm6bMzExFRkaqT58+GjdunDeaAQAASimvBJS7775b7q69jYyM1IYNG7yxSQAAUIbwWzwAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByfEu6AQBwLdUe+5VX1ps8qbtX1gtcrxhBAQAAllPsAWXixImy2WxOU8OGDR3zz58/r5EjR6pKlSqqVKmS+vTpo9TU1OJuBgAAKMW8MoLSuHFjHTt2zDF98803jnmjR4/WF198ocWLF2vDhg06evSo7rvvPm80AwAAlFJeuQbF19dXYWFhLuXp6emaNWuW5s+fr7vuukuSNHv2bDVq1EibN2/W7bff7o3mAACAUsYrIyj79u1TRESE6tatqwEDBujw4cOSpO3bt+vChQvq3Lmzo27Dhg1Vs2ZNJSYmFri+rKwsZWRkOE0AAKDsKvYRlOjoaM2ZM0cNGjTQsWPHFB8fr7Zt2+qHH35QSkqK/Pz8VLlyZadlqlevrpSUlALXmZCQoPj4+OJuKkqYt+6mAACUfsUeUGJjYx1/N23aVNHR0apVq5Y+/vhjBQQEXNE64+LiNGbMGMfjjIwMRUZGXnVbAQCANXn9NuPKlSurfv362r9/v8LCwpSdna20tDSnOqmpqW6vWclnt9sVFBTkNAEAgLLL6wHl7NmzOnDggMLDw9WyZUuVL19ea9asccxPSkrS4cOH1bp1a283BQAAlBLFforn6aefVs+ePVWrVi0dPXpUEyZMULly5dSvXz8FBwdryJAhGjNmjEJCQhQUFKQnn3xSrVu35g4eAADgUOwB5ddff1W/fv106tQpVatWTW3atNHmzZtVrVo1SdIbb7whHx8f9enTR1lZWYqJidE777xT3M0AAAClmM0YY0q6EZ7KyMhQcHCw0tPTuR6lFOMuHpQl/BYPcHmefH7zWzwAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByij2gJCQk6E9/+pMCAwMVGhqq3r17KykpyalOhw4dZLPZnKYRI0YUd1MAAEApVewBZcOGDRo5cqQ2b96sVatW6cKFC7r77ruVmZnpVG/o0KE6duyYY3rttdeKuykAAKCU8i3uFS5fvtzp8Zw5cxQaGqrt27erXbt2jvIKFSooLCysSOvMyspSVlaW43FGRkbxNBYAAFiS169BSU9PlySFhIQ4lc+bN09Vq1bVLbfcori4OJ07d67AdSQkJCg4ONgxRUZGerXNAACgZNmMMcZbK8/Ly1OvXr2Ulpamb775xlE+c+ZM1apVSxEREdq9e7eee+45tWrVSkuXLnW7HncjKJGRkUpPT1dQUJC3mg8vqz32q5JuAlBskid1L+kmAJaXkZGh4ODgIn1+F/spnouNHDlSP/zwg1M4kaRhw4Y5/m7SpInCw8PVqVMnHThwQFFRUS7rsdvtstvt3mwqAFwVbwZuwg+uR147xfPEE0/oyy+/1Lp163TjjTcWWjc6OlqStH//fm81BwAAlCLFPoJijNGTTz6pTz/9VOvXr1edOnUuu8yuXbskSeHh4cXdHAAAUAoVe0AZOXKk5s+fr88++0yBgYFKSUmRJAUHBysgIEAHDhzQ/Pnz1a1bN1WpUkW7d+/W6NGj1a5dOzVt2rS4mwMAAEqhYg8o7777rqQ/voztYrNnz9ZDDz0kPz8/rV69WtOmTVNmZqYiIyPVp08fjRs3rribAgAASimvnOIpTGRkpDZs2FDcmwUAAGUIv8UDAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsx6u/ZgwAuHre+qVkfiUZVsYICgAAsBwCCgAAsBwCCgAAsByuQUGhvHXuGwCAwjCCAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIfvQQGA65Q3v+eI3/nB1WIEBQAAWA4BBQAAWA4BBQAAWA4BBQAAWA4BBQAAWA538QAAip237hDi7qDrByMoAADAcggoAADAcggoAADAcrgGpYzw5jdCAgBwrTGCAgAALIeAAgAALIeAAgAALIeAAgAALIeLZAEApYY3bwjgS+CshREUAABgOSU6gjJ9+nRNmTJFKSkpatasmd566y21atWqJJsEALhOlcav5y/LI0olNoKyaNEijRkzRhMmTNCOHTvUrFkzxcTE6Pjx4yXVJAAAYBE2Y4wpiQ1HR0frT3/6k95++21JUl5eniIjI/Xkk09q7NixhS6bkZGh4OBgpaenKygoqNjbxpeeAQCud94YQfHk87tETvFkZ2dr+/btiouLc5T5+Pioc+fOSkxMdKmflZWlrKwsx+P09HRJf+yoN+RlnfPKegEAKC288Rmbv86ijI2USEA5efKkcnNzVb16dafy6tWr66effnKpn5CQoPj4eJfyyMhIr7URAIDrWfA07637zJkzCg4OLrROqbjNOC4uTmPGjHE8zsvL0+nTp1WlShXZbLZi2UZGRoYiIyP1yy+/eOW00fWG/ixe9Gfxoj+LF/1ZvMpyfxpjdObMGUVERFy2bokElKpVq6pcuXJKTU11Kk9NTVVYWJhLfbvdLrvd7lRWuXJlr7QtKCiozB0QJYn+LF70Z/GiP4sX/Vm8ymp/Xm7kJF+J3MXj5+enli1bas2aNY6yvLw8rVmzRq1bty6JJgEAAAspsVM8Y8aM0eDBg3XbbbepVatWmjZtmjIzM/Xwww+XVJMAAIBFlFhAuf/++3XixAm9+OKLSklJUfPmzbV8+XKXC2evFbvdrgkTJricSsKVoT+LF/1ZvOjP4kV/Fi/68w8l9j0oAAAABeG3eAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOVcNwFl+vTpql27tvz9/RUdHa3//Oc/hdZPS0vTyJEjFR4eLrvdrvr16+vrr7++Rq0tHTzt02nTpqlBgwYKCAhQZGSkRo8erfPnz1+j1lrXxo0b1bNnT0VERMhms2nZsmWXXWb9+vW69dZbZbfbddNNN2nOnDleb2dp4Wl/Ll26VF26dFG1atUUFBSk1q1ba8WKFdemsaXElRyj+b799lv5+vqqefPmXmtfaXMl/ZmVlaUXXnhBtWrVkt1uV+3atfXBBx94v7El6LoIKIsWLdKYMWM0YcIE7dixQ82aNVNMTIyOHz/utn52dra6dOmi5ORkffLJJ0pKStL777+vGjVqXOOWW5enfTp//nyNHTtWEyZM0I8//qhZs2Zp0aJFev75569xy60nMzNTzZo10/Tp04tU/9ChQ+revbs6duyoXbt26amnntKjjz7Kh+r/52l/bty4UV26dNHXX3+t7du3q2PHjurZs6d27tzp5ZaWHp72ab60tDQ9+OCD6tSpk5daVjpdSX/27dtXa9as0axZs5SUlKQFCxaoQYMGXmylBZjrQKtWrczIkSMdj3Nzc01ERIRJSEhwW//dd981devWNdnZ2deqiaWOp306cuRIc9dddzmVjRkzxtx5551ebWdpI8l8+umnhdZ59tlnTePGjZ3K7r//fhMTE+PFlpVORelPd26++WYTHx9f/A0qAzzp0/vvv9+MGzfOTJgwwTRr1syr7SqtitKf//73v01wcLA5derUtWmURZT5EZTs7Gxt375dnTt3dpT5+Pioc+fOSkxMdLvM559/rtatW2vkyJGqXr26brnlFv3tb39Tbm7utWq2pV1Jn95xxx3avn274zTQwYMH9fXXX6tbt27XpM1lSWJiolPfS1JMTEyBfQ/P5OXl6cyZMwoJCSnpppRqs2fP1sGDBzVhwoSSbkqp9/nnn+u2227Ta6+9pho1aqh+/fp6+umn9fvvv5d007yqxL7q/lo5efKkcnNzXb5Cv3r16vrpp5/cLnPw4EGtXbtWAwYM0Ndff639+/fr8ccf14ULF3ix6cr6tH///jp58qTatGkjY4xycnI0YsQITvFcgZSUFLd9n5GRod9//10BAQEl1LKyYerUqTp79qz69u1b0k0ptfbt26exY8dq06ZN8vUt8x8zXnfw4EF988038vf316effqqTJ0/q8ccf16lTpzR79uySbp7XlPkRlCuRl5en0NBQzZw5Uy1bttT999+vF154Qe+9915JN63UWr9+vf72t7/pnXfe0Y4dO7R06VJ99dVXevnll0u6aYDD/PnzFR8fr48//lihoaEl3ZxSKTc3V/3791d8fLzq169f0s0pE/Ly8mSz2TRv3jy1atVK3bp10+uvv64PP/ywTI+ilPloW7VqVZUrV06pqalO5ampqQoLC3O7THh4uMqXL69y5co5yho1aqSUlBRlZ2fLz8/Pq222uivp0/Hjx2vQoEF69NFHJUlNmjRRZmamhg0bphdeeEE+PmTlogoLC3Pb90FBQYyeXIWFCxfq0Ucf1eLFi11OoaHozpw5o23btmnnzp164oknJP3xAWuMka+vr1auXKm77rqrhFtZuoSHh6tGjRoKDg52lDVq1EjGGP3666+qV69eCbbOe8r8p4Kfn59atmypNWvWOMry8vK0Zs0atW7d2u0yd955p/bv36+8vDxH2d69exUeHn7dhxPpyvr03LlzLiEkPwAafq/SI61bt3bqe0latWpVgX2Py1uwYIEefvhhLViwQN27dy/p5pRqQUFB+v7777Vr1y7HNGLECDVo0EC7du1SdHR0STex1Lnzzjt19OhRnT171lG2d+9e+fj46MYbbyzBlnlZyV6je20sXLjQ2O12M2fOHPO///3PDBs2zFSuXNmkpKQYY4wZNGiQGTt2rKP+4cOHTWBgoHniiSdMUlKS+fLLL01oaKh55ZVXSmoXLMfTPp0wYYIJDAw0CxYsMAcPHjQrV640UVFRpm/fviW1C5Zx5swZs3PnTrNz504jybz++utm586d5ueffzbGGDN27FgzaNAgR/2DBw+aChUqmGeeecb8+OOPZvr06aZcuXJm+fLlJbULluJpf86bN8/4+vqa6dOnm2PHjjmmtLS0ktoFy/G0Ty/FXTzOPO3PM2fOmBtvvNH8+c9/Nnv27DEbNmww9erVM48++mhJ7cI1cV0EFGOMeeutt0zNmjWNn5+fadWqldm8ebNjXvv27c3gwYOd6n/33XcmOjra2O12U7duXfPqq6+anJyca9xqa/OkTy9cuGAmTpxooqKijL+/v4mMjDSPP/64+e233659wy1m3bp1RpLLlN9/gwcPNu3bt3dZpnnz5sbPz8/UrVvXzJ49+5q326o87c/27dsXWh9XdoxejIDi7Er688cffzSdO3c2AQEB5sYbbzRjxowx586du/aNv4ZsxjC+DgAArKXMX4MCAABKHwIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwnP8HSRVOKMsLUaUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(adequacy_score, bins = 20)\n",
    "plt.title('Отношение длины перевода на EN к референсу EN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "842c2973-1b5a-49f6-bab7-f1317a4fc860",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:58:20.614286Z",
     "iopub.status.busy": "2024-10-18T06:58:20.614004Z",
     "iopub.status.idle": "2024-10-18T06:58:20.624732Z",
     "shell.execute_reply": "2024-10-18T06:58:20.624058Z",
     "shell.execute_reply.started": "2024-10-18T06:58:20.614267Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from typing import Dict, List\n",
    "from comet import download_model, load_from_checkpoint\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "81931529-be02-4c34-ab03-3fcc776cce60",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:58:21.373607Z",
     "iopub.status.busy": "2024-10-18T06:58:21.372532Z",
     "iopub.status.idle": "2024-10-18T06:59:07.100618Z",
     "shell.execute_reply": "2024-10-18T06:59:07.099757Z",
     "shell.execute_reply.started": "2024-10-18T06:58:21.373585Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching 5 files: 100%|██████████| 5/5 [00:00<00:00, 53092.46it/s]\n",
      "Encoder model frozen.\n",
      "/home/jupyter/.local/lib/python3.10/site-packages/pytorch_lightning/core/saving.py:195: Found keys that are not in the model state dict but in the checkpoint: ['encoder.model.embeddings.position_ids']\n"
     ]
    }
   ],
   "source": [
    "model_path_xcomet = download_model(\"Unbabel/XCOMET-XL\")\n",
    "model_xcomet = load_from_checkpoint(model_path_xcomet)\n",
    "model_xcomet = model_xcomet.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3481da04-740b-401c-bd02-a699a139c03e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:59:07.101994Z",
     "iopub.status.busy": "2024-10-18T06:59:07.101703Z",
     "iopub.status.idle": "2024-10-18T06:59:07.118208Z",
     "shell.execute_reply": "2024-10-18T06:59:07.117514Z",
     "shell.execute_reply.started": "2024-10-18T06:59:07.101975Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def xcomet(data: List[Dict[str, str]]) -> List[float]:\n",
    "    \n",
    "    '''Format\n",
    "    data = [\n",
    "    {\n",
    "        # Source, текст, который надо перевести, src\n",
    "        \"src\": \"В понедельник\", \n",
    "        \n",
    "        # Machine Translation\n",
    "        \"mt\": \"On Monday\", \n",
    "        \n",
    "        # Эталонный перевод, en\n",
    "        \"ref\": \"On Monday\" \n",
    "    }'''\n",
    "    \n",
    "    \n",
    "    xcomet_metric = model_xcomet.predict(data, batch_size=8, gpus=1)\n",
    "    return xcomet_metric.scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "676e1841-fcea-4cdb-9bd8-912cf7e1d032",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:59:25.054393Z",
     "iopub.status.busy": "2024-10-18T06:59:25.054028Z",
     "iopub.status.idle": "2024-10-18T07:01:18.990479Z",
     "shell.execute_reply": "2024-10-18T07:01:18.989665Z",
     "shell.execute_reply.started": "2024-10-18T06:59:25.054372Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "Predicting DataLoader 0: 100%|██████████| 127/127 [01:47<00:00,  1.18it/s]\n"
     ]
    }
   ],
   "source": [
    "xcomet_cpo_score = xcomet(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "71f927bf-3522-450e-a485-86ba6e38a7b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:01:18.992617Z",
     "iopub.status.busy": "2024-10-18T07:01:18.991845Z",
     "iopub.status.idle": "2024-10-18T07:01:19.193894Z",
     "shell.execute_reply": "2024-10-18T07:01:19.193249Z",
     "shell.execute_reply.started": "2024-10-18T07:01:18.992594Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'XCOMET')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGzCAYAAAAFROyYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqHUlEQVR4nO3df1xUdaL/8TegDPhjhtVkgMJS05S0LEyctM1rXEnRR11x0zSjfbh5c9G9Slp681foitduarb+qG4rtum6WdqaP1CiVXPFH1E9rr/LtAulg6XBqJugcL5/7IP5NoHpIMhn6PV8PM7jEed8zpnPOdHOa2fODEGWZVkCAAAwSHB9TwAAAODHCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFgI/HHntMYWFh+uyzz6psmzNnjoKCgrR+/XpJUlFRkSZMmKCOHTuqSZMmatq0qeLj4zVr1iwVFxf77GtZlv70pz/pl7/8pSIiItSkSRN16dJFGRkZOn/+fJXH6t27t4KCgtS+fftq55mTk6OgoCAFBQXp7bff9q7Pysryrq9u2bVrl5544omfHFO5PPHEEzW/kACuSaP6ngAAs8ybN08bN27UU089pQ8++MC7/vjx48rIyFBKSooGDBigvXv3qn///jp37pwee+wxxcfHS5I++ugjzZkzR9u3b9eWLVskSeXl5Ro2bJjeeust3XfffZoxY4aaNGmiDz/8UM8//7xWr16t999/X06n02cuYWFhOnr0qPbs2aPu3bv7bFuxYoXCwsJ04cKFas8jIyNDbdq0qbL+1ltv1b//+78rMTHR59ymTZumUaNG6b777vOub9eunZ9XD0CtsQDgR1599VVLkpWVleVd9+CDD1p2u9366quvrO+++8668cYbLafTaR06dKjK/m6325o5c6b359mzZ1uSrAkTJlQZu27dOis4ONh68MEHfdbff//91u23327ddttt1rhx43y2ff/995bdbrdSUlIsSdbq1au925YtW2ZJsvbu3XvV57t3715LkrVs2bKr3gdA3eItHgBV/OY3v1HPnj01YcIEnT59WqtWrVJ2drZmzZqlG2+8Ua+88oq+/vprzZs3Tx07dqyyv9Pp1JQpUyRJ33//vV544QV16NBBmZmZVcYOHDhQqampys7O1q5du6psf/TRR/WXv/xFFRUV3nXvvfee/vGPf+iRRx6pxbMGYBICBUAVQUFBeuWVV1RSUqLRo0dr/Pjx6tatm9LS0iRJ69atU3h4uAYPHnzFY+3YsUPfffedhg0bpkaNqn9X+fHHH5ck770tPzRs2DCdPHlSW7du9a5buXKlHnjgAUVGRl72cUtKSvTtt9/6LKdPn77ifAGYgXtQAFTr9ttv14QJE5SZmamQkBBt2LBBwcH//P80hw4dUocOHRQaGnrF4xw8eFCSdOedd152TOW2Q4cOVdnWvn17devWTStXrlSfPn1UXFysjRs36rXXXvvJx/3hPSaVbDbbZe9ZAWAWAgXAZd1www2SpJiYGHXu3Nm73uPxqHnz5ld1jLNnz0rST46v3ObxeKrdPmzYMM2cOVOLFy/W22+/rZCQEP3bv/2b8vPzL3vMRYsWqUOHDj7rQkJCrmrOAOofgQKgWoWFhZo+fbo6d+6s/fv3a+7cud77Sux2uzc8rqQyPn5q/JUiZujQoZowYYI2bdqkFStWaMCAAVcMpO7du6tbt25XNUcA5uEeFADVGjNmjCRp06ZN+tWvfqXf//73OnbsmCSpY8eO+uyzz1RWVnbF43Tq1EmS9L//+7+XHVO5LS4urtrt0dHR6t27t1588UVt375dw4YN8+tcAAQeAgVAFWvXrtW6des0c+ZM3XTTTVqwYIFCQ0O9N8kOHDhQ33//vd55550rHqtXr16KiIjQypUrVV5eXu2YN954Q5I0YMCAyx5n2LBh+vDDD2W329W/f/8anBWAQEKgAPBx9uxZ/e53v9Ndd92lsWPHSvrnPSgzZ85Udna2Vq9eraeeekrR0dF6+umnq/3G2VOnTmnWrFmSpCZNmmjChAk6cuSInnvuuSpjN2zYoKysLCUlJalHjx6XndfgwYM1ffp0LV68+KpuzgUQ2LgHBYCPKVOm6MSJE1qzZo3PTaVpaWlavny5xo0bp8OHD2vt2rXq37+/unbt6vNNsh9//LH+/Oc/y+VyefedNGmSPvnkE/3Xf/2X8vLylJKSovDwcO3YsUNvvvmmOnXqpOXLl//kvBwOh2bMmHHV57Fp0yYdPny4yvp7771Xbdu2verjAKgfBAoAr/z8fC1atEi//e1vdc899/hsCwkJ0dKlS9WjRw9NmTJFL730kvbv368XXnhBGzZs0J/+9CcFBwerU6dOmjRpkvcelsp933rrLb3xxhv6n//5H02dOlVlZWVq166dpk+frqefflpNmzat1XOZNm1ateuXLVtGoAABIMiyLKu+JwEAAPBD3IMCAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOME5PegVFRU6MSJE2revLmCgoLqezoAAOAqWJals2fPKiYmRsHBP/0aSUAGyokTJxQbG1vf0wAAADVQWFiom2666SfHBGSgVP6Z9cLCQtnt9nqeDQAAuBoej0exsbHe5/GfEpCBUvm2jt1uJ1AAAAgwV3N7BjfJAgAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOI3qewIAAKBmbpm0oc6O/eWc5Do79tXgFRQAAGAcAgUAABiHQAEAAMbxO1C+/vprPfbYY2rZsqXCw8PVpUsXffTRR97tlmVp2rRpio6OVnh4uBITE/X555/7HOPMmTMaPny47Ha7IiIiNHLkSJ07d+7azwYAADQIfgXKd999p549e6px48batGmTDh48qBdffFG/+MUvvGPmzp2rhQsXaunSpdq9e7eaNm2qpKQkXbhwwTtm+PDhOnDggHJycrR+/Xpt375do0aNqr2zAgAAAS3IsizragdPmjRJf//73/Xhhx9Wu92yLMXExOjpp5/WhAkTJEklJSVyOp3KysrS0KFDdejQIcXFxWnv3r3q1q2bJCk7O1v9+/fXV199pZiYmCvOw+PxyOFwqKSkRHa7/WqnDwBAgxJon+Lx5/nbr1dQ1q1bp27duulXv/qVIiMjddddd+m1117zbj9+/LjcbrcSExO96xwOhxISEpSXlydJysvLU0REhDdOJCkxMVHBwcHavXt3tY9bWloqj8fjswAAgIbLr0A5duyYlixZovbt22vz5s0aPXq0fve732n58uWSJLfbLUlyOp0++zmdTu82t9utyMhIn+2NGjVSixYtvGN+LDMzUw6Hw7vExsb6M20AABBg/AqUiooK3X333Zo9e7buuusujRo1Sk8++aSWLl1aV/OTJE2ePFklJSXepbCwsE4fDwAA1C+/AiU6OlpxcXE+6zp16qSCggJJUlRUlCSpqKjIZ0xRUZF3W1RUlE6dOuWz/dKlSzpz5ox3zI/ZbDbZ7XafBQAANFx+BUrPnj115MgRn3WfffaZbr75ZklSmzZtFBUVpdzcXO92j8ej3bt3y+VySZJcLpeKi4uVn5/vHfPBBx+ooqJCCQkJNT4RAADQcPj1t3jGjx+ve++9V7Nnz9YjjzyiPXv26NVXX9Wrr74qSQoKCtK4ceM0a9YstW/fXm3atNHUqVMVExOjhx9+WNI/X3F58MEHvW8NXbx4UWPGjNHQoUOv6hM8AACg4fMrUO655x6tXbtWkydPVkZGhtq0aaMFCxZo+PDh3jHPPPOMzp8/r1GjRqm4uFi9evVSdna2wsLCvGNWrFihMWPG6IEHHlBwcLBSUlK0cOHC2jsrAAAQ0Pz6HhRT8D0oAADwPSgAAADXFYECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjONXoMyYMUNBQUE+S8eOHb3bL1y4oLS0NLVs2VLNmjVTSkqKioqKfI5RUFCg5ORkNWnSRJGRkZo4caIuXbpUO2cDAAAahEb+7nD77bfr/fff//8HaPT/DzF+/Hht2LBBq1evlsPh0JgxYzRo0CD9/e9/lySVl5crOTlZUVFR2rlzp06ePKnHH39cjRs31uzZs2vhdAAAQEPgd6A0atRIUVFRVdaXlJTo9ddf18qVK9WnTx9J0rJly9SpUyft2rVLPXr00JYtW3Tw4EG9//77cjqd6tq1q2bOnKlnn31WM2bMUGho6LWfEQAACHh+34Py+eefKyYmRm3bttXw4cNVUFAgScrPz9fFixeVmJjoHduxY0e1bt1aeXl5kqS8vDx16dJFTqfTOyYpKUkej0cHDhy47GOWlpbK4/H4LAAAoOHyK1ASEhKUlZWl7OxsLVmyRMePH9d9992ns2fPyu12KzQ0VBERET77OJ1Oud1uSZLb7faJk8rtldsuJzMzUw6Hw7vExsb6M20AABBg/HqLp1+/ft5/vuOOO5SQkKCbb75Zb731lsLDw2t9cpUmT56s9PR0788ej4dIAQCgAbumjxlHRESoQ4cOOnr0qKKiolRWVqbi4mKfMUVFRd57VqKioqp8qqfy5+rua6lks9lkt9t9FgAA0HBdU6CcO3dOX3zxhaKjoxUfH6/GjRsrNzfXu/3IkSMqKCiQy+WSJLlcLu3bt0+nTp3yjsnJyZHdbldcXNy1TAUAADQgfr3FM2HCBA0cOFA333yzTpw4oenTpyskJESPPvqoHA6HRo4cqfT0dLVo0UJ2u11jx46Vy+VSjx49JEl9+/ZVXFycRowYoblz58rtdmvKlClKS0uTzWarkxMEAACBx69A+eqrr/Too4/q9OnTatWqlXr16qVdu3apVatWkqT58+crODhYKSkpKi0tVVJSkhYvXuzdPyQkROvXr9fo0aPlcrnUtGlTpaamKiMjo3bPCgAABLQgy7Ks+p6EvzwejxwOh0pKSrgfBQDws3XLpA11duwv5yTX+jH9ef7mb/EAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwzjUFypw5cxQUFKRx48Z51124cEFpaWlq2bKlmjVrppSUFBUVFfnsV1BQoOTkZDVp0kSRkZGaOHGiLl26dC1TAQAADUiNA2Xv3r165ZVXdMcdd/isHz9+vN577z2tXr1a27Zt04kTJzRo0CDv9vLyciUnJ6usrEw7d+7U8uXLlZWVpWnTptX8LAAAQINSo0A5d+6chg8frtdee02/+MUvvOtLSkr0+uuva968eerTp4/i4+O1bNky7dy5U7t27ZIkbdmyRQcPHtSbb76prl27ql+/fpo5c6YWLVqksrKy2jkrAAAQ0GoUKGlpaUpOTlZiYqLP+vz8fF28eNFnfceOHdW6dWvl5eVJkvLy8tSlSxc5nU7vmKSkJHk8Hh04cKDaxystLZXH4/FZAABAw9XI3x1WrVqljz/+WHv37q2yze12KzQ0VBERET7rnU6n3G63d8wP46Rye+W26mRmZur555/3d6oAACBA+fUKSmFhof7jP/5DK1asUFhYWF3NqYrJkyerpKTEuxQWFl63xwYAANefX4GSn5+vU6dO6e6771ajRo3UqFEjbdu2TQsXLlSjRo3kdDpVVlam4uJin/2KiooUFRUlSYqKiqryqZ7KnyvH/JjNZpPdbvdZAABAw+VXoDzwwAPat2+fPv30U+/SrVs3DR8+3PvPjRs3Vm5urnefI0eOqKCgQC6XS5Lkcrm0b98+nTp1yjsmJydHdrtdcXFxtXRaAAAgkPl1D0rz5s3VuXNnn3VNmzZVy5YtvetHjhyp9PR0tWjRQna7XWPHjpXL5VKPHj0kSX379lVcXJxGjBihuXPnyu12a8qUKUpLS5PNZqul0wIAAIHM75tkr2T+/PkKDg5WSkqKSktLlZSUpMWLF3u3h4SEaP369Ro9erRcLpeaNm2q1NRUZWRk1PZUAABAgAqyLMuq70n4y+PxyOFwqKSkhPtRAAA/W7dM2lBnx/5yTnKtH9Of52/+Fg8AADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACM41egLFmyRHfccYfsdrvsdrtcLpc2bdrk3X7hwgWlpaWpZcuWatasmVJSUlRUVORzjIKCAiUnJ6tJkyaKjIzUxIkTdenSpdo5GwAA0CD4FSg33XST5syZo/z8fH300Ufq06ePHnroIR04cECSNH78eL333ntavXq1tm3bphMnTmjQoEHe/cvLy5WcnKyysjLt3LlTy5cvV1ZWlqZNm1a7ZwUAAAJakGVZ1rUcoEWLFnrhhRc0ePBgtWrVSitXrtTgwYMlSYcPH1anTp2Ul5enHj16aNOmTRowYIBOnDghp9MpSVq6dKmeffZZffPNNwoNDb2qx/R4PHI4HCopKZHdbr+W6QMAELBumbShzo795ZzkWj+mP8/fNb4Hpby8XKtWrdL58+flcrmUn5+vixcvKjEx0TumY8eOat26tfLy8iRJeXl56tKlizdOJCkpKUkej8f7Kkx1SktL5fF4fBYAANBw+R0o+/btU7NmzWSz2fTUU09p7dq1iouLk9vtVmhoqCIiInzGO51Oud1uSZLb7faJk8rtldsuJzMzUw6Hw7vExsb6O20AABBA/A6U2267TZ9++ql2796t0aNHKzU1VQcPHqyLuXlNnjxZJSUl3qWwsLBOHw8AANSvRv7uEBoaqltvvVWSFB8fr7179+qll17SkCFDVFZWpuLiYp9XUYqKihQVFSVJioqK0p49e3yOV/kpn8ox1bHZbLLZbP5OFQAABKhr/h6UiooKlZaWKj4+Xo0bN1Zubq5325EjR1RQUCCXyyVJcrlc2rdvn06dOuUdk5OTI7vdrri4uGudCgAAaCD8egVl8uTJ6tevn1q3bq2zZ89q5cqV2rp1qzZv3iyHw6GRI0cqPT1dLVq0kN1u19ixY+VyudSjRw9JUt++fRUXF6cRI0Zo7ty5crvdmjJlitLS0niFBAAAePkVKKdOndLjjz+ukydPyuFw6I477tDmzZv1r//6r5Kk+fPnKzg4WCkpKSotLVVSUpIWL17s3T8kJETr16/X6NGj5XK51LRpU6WmpiojI6N2zwoAAAS0a/4elPrA96AAAMD3oAAAAFxXBAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACM41egZGZm6p577lHz5s0VGRmphx9+WEeOHPEZc+HCBaWlpally5Zq1qyZUlJSVFRU5DOmoKBAycnJatKkiSIjIzVx4kRdunTp2s8GAAA0CH4FyrZt25SWlqZdu3YpJydHFy9eVN++fXX+/HnvmPHjx+u9997T6tWrtW3bNp04cUKDBg3ybi8vL1dycrLKysq0c+dOLV++XFlZWZo2bVrtnRUAAAhoQZZlWTXd+ZtvvlFkZKS2bdumX/7ylyopKVGrVq20cuVKDR48WJJ0+PBhderUSXl5eerRo4c2bdqkAQMG6MSJE3I6nZKkpUuX6tlnn9U333yj0NDQKo9TWlqq0tJS788ej0exsbEqKSmR3W6v6fQBAAhot0zaUGfH/nJOcq0f0+PxyOFwXNXz9zXdg1JSUiJJatGihSQpPz9fFy9eVGJiondMx44d1bp1a+Xl5UmS8vLy1KVLF2+cSFJSUpI8Ho8OHDhQ7eNkZmbK4XB4l9jY2GuZNgAAMFyNA6WiokLjxo1Tz5491blzZ0mS2+1WaGioIiIifMY6nU653W7vmB/GSeX2ym3VmTx5skpKSrxLYWFhTacNAAACQKOa7piWlqb9+/drx44dtTmfatlsNtlstjp/HAAAYIYavYIyZswYrV+/Xn/729900003eddHRUWprKxMxcXFPuOLiooUFRXlHfPjT/VU/lw5BgAA/Lz5FSiWZWnMmDFau3atPvjgA7Vp08Zne3x8vBo3bqzc3FzvuiNHjqigoEAul0uS5HK5tG/fPp06dco7JicnR3a7XXFxcddyLgAAoIHw6y2etLQ0rVy5Un/961/VvHlz7z0jDodD4eHhcjgcGjlypNLT09WiRQvZ7XaNHTtWLpdLPXr0kCT17dtXcXFxGjFihObOnSu3260pU6YoLS2Nt3EAAIAkPwNlyZIlkqTevXv7rF+2bJmeeOIJSdL8+fMVHByslJQUlZaWKikpSYsXL/aODQkJ0fr16zV69Gi5XC41bdpUqampysjIuLYzAQAADcY1fQ9KffHnc9QAADRUfA8KAADAdUSgAAAA4xAoAADAODX+ojYAAHBldXmfSEPGKygAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMI7fgbJ9+3YNHDhQMTExCgoK0rvvvuuz3bIsTZs2TdHR0QoPD1diYqI+//xznzFnzpzR8OHDZbfbFRERoZEjR+rcuXPXdCIAAKDh8DtQzp8/rzvvvFOLFi2qdvvcuXO1cOFCLV26VLt371bTpk2VlJSkCxcueMcMHz5cBw4cUE5OjtavX6/t27dr1KhRNT8LAADQoDTyd4d+/fqpX79+1W6zLEsLFizQlClT9NBDD0mS3njjDTmdTr377rsaOnSoDh06pOzsbO3du1fdunWTJL388svq37+//vu//1sxMTHXcDoAAKAhqNV7UI4fPy63263ExETvOofDoYSEBOXl5UmS8vLyFBER4Y0TSUpMTFRwcLB2795d7XFLS0vl8Xh8FgAA0HDVaqC43W5JktPp9FnvdDq929xutyIjI322N2rUSC1atPCO+bHMzEw5HA7vEhsbW5vTBgAAhgmIT/FMnjxZJSUl3qWwsLC+pwQAAOpQrQZKVFSUJKmoqMhnfVFRkXdbVFSUTp065bP90qVLOnPmjHfMj9lsNtntdp8FAAA0XLUaKG3atFFUVJRyc3O96zwej3bv3i2XyyVJcrlcKi4uVn5+vnfMBx98oIqKCiUkJNTmdAAAQIDy+1M8586d09GjR70/Hz9+XJ9++qlatGih1q1ba9y4cZo1a5bat2+vNm3aaOrUqYqJidHDDz8sSerUqZMefPBBPfnkk1q6dKkuXryoMWPGaOjQoXyCBwAASKpBoHz00Uf6l3/5F+/P6enpkqTU1FRlZWXpmWee0fnz5zVq1CgVFxerV69eys7OVlhYmHefFStWaMyYMXrggQcUHByslJQULVy4sBZOBwAANARBlmVZ9T0Jf3k8HjkcDpWUlHA/CgDAaLdM2lDfU6iRL+ck1/ox/Xn+DohP8QAAgJ8XAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYx++/xQMAQEMUqF9J31DxCgoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4/LFAAEDA4A/6/XzwCgoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOHxRGwCg1vGFarhWvIICAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOPwMWMA+Jnio8AwGa+gAAAA4xAoAADAOAQKAAAwDvegAEAtqMv7Ob6ck1xnxwZMRaAAgOG4mRU/R7zFAwAAjFOvgbJo0SLdcsstCgsLU0JCgvbs2VOf0wEAAIaot7d4/vKXvyg9PV1Lly5VQkKCFixYoKSkJB05ckSRkZH1NS0AfuCtBwB1JciyLKs+HjghIUH33HOP/vCHP0iSKioqFBsbq7Fjx2rSpEk/ua/H45HD4VBJSYnsdvv1mC4QsIgIADVRFzdn+/P8XS+voJSVlSk/P1+TJ0/2rgsODlZiYqLy8vKqjC8tLVVpaan355KSEkn/PFEAP62i9B/1PQUAAagunmMrj3k1r43US6B8++23Ki8vl9Pp9FnvdDp1+PDhKuMzMzP1/PPPV1kfGxtbZ3MEAODnzLGg7o599uxZORyOnxwTEB8znjx5stLT070/V1RU6MyZM2rZsqWCgoLqZU4ej0exsbEqLCzkbaY6wPWtW1zfusX1rVtc37pXV9fYsiydPXtWMTExVxxbL4Fyww03KCQkREVFRT7ri4qKFBUVVWW8zWaTzWbzWRcREVGXU7xqdrud/0DqENe3bnF96xbXt25xfeteXVzjK71yUqlePmYcGhqq+Ph45ebmetdVVFQoNzdXLperPqYEAAAMUm9v8aSnpys1NVXdunVT9+7dtWDBAp0/f16//vWv62tKAADAEPUWKEOGDNE333yjadOmye12q2vXrsrOzq5y46ypbDabpk+fXuWtJ9QOrm/d4vrWLa5v3eL61j0TrnG9fQ8KAADA5fC3eAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQiUn7Bo0SLdcsstCgsLU0JCgvbs2XPZsVlZWQoKCvJZwsLCruNsA48/11eSiouLlZaWpujoaNlsNnXo0EEbN268TrMNPP5c3969e1f5/Q0KClJycu3/NdOGwt/f3wULFui2225TeHi4YmNjNX78eF24cOE6zTbw+HN9L168qIyMDLVr105hYWG68847lZ2dfR1nG1i2b9+ugQMHKiYmRkFBQXr33XevuM/WrVt19913y2az6dZbb1VWVladz1MWqrVq1SorNDTU+uMf/2gdOHDAevLJJ62IiAirqKio2vHLli2z7Ha7dfLkSe/idruv86wDh7/Xt7S01OrWrZvVv39/a8eOHdbx48etrVu3Wp9++ul1nnlg8Pf6nj592ud3d//+/VZISIi1bNmy6zvxAOHv9V2xYoVls9msFStWWMePH7c2b95sRUdHW+PHj7/OMw8M/l7fZ555xoqJibE2bNhgffHFF9bixYutsLAw6+OPP77OMw8MGzdutJ577jlrzZo1liRr7dq1Pzn+2LFjVpMmTaz09HTr4MGD1ssvv2yFhIRY2dnZdTpPAuUyunfvbqWlpXl/Li8vt2JiYqzMzMxqxy9btsxyOBzXaXaBz9/ru2TJEqtt27ZWWVnZ9ZpiQPP3+v7Y/PnzrebNm1vnzp2rqykGNH+vb1pamtWnTx+fdenp6VbPnj3rdJ6Byt/rGx0dbf3hD3/wWTdo0CBr+PDhdTrPhuBqAuWZZ56xbr/9dp91Q4YMsZKSkupwZpbFWzzVKCsrU35+vhITE73rgoODlZiYqLy8vMvud+7cOd18882KjY3VQw89pAMHDlyP6QacmlzfdevWyeVyKS0tTU6nU507d9bs2bNVXl5+vaYdMGr6+/tDr7/+uoYOHaqmTZvW1TQDVk2u77333qv8/Hzv2xTHjh3Txo0b1b9//+sy50BSk+tbWlpa5S318PBw7dixo07n+nORl5fn8+9DkpKSkq76f09qikCpxrfffqvy8vIqX7vvdDrldrur3ee2227TH//4R/31r3/Vm2++qYqKCt1777366quvrseUA0pNru+xY8f09ttvq7y8XBs3btTUqVP14osvatasWddjygGlJtf3h/bs2aP9+/frN7/5TV1NMaDV5PoOGzZMGRkZ6tWrlxo3bqx27dqpd+/e+s///M/rMeWAUpPrm5SUpHnz5unzzz9XRUWFcnJytGbNGp08efJ6TLnBc7vd1f778Hg8+v777+vscQmUWuJyufT444+ra9euuv/++7VmzRq1atVKr7zySn1PrUGoqKhQZGSkXn31VcXHx2vIkCF67rnntHTp0vqeWoPz+uuvq0uXLurevXt9T6XB2Lp1q2bPnq3Fixfr448/1po1a7RhwwbNnDmzvqfWILz00ktq3769OnbsqNDQUI0ZM0a//vWvFRzMU1wgq7c/FmiyG264QSEhISoqKvJZX1RUpKioqKs6RuPGjXXXXXfp6NGjdTHFgFaT6xsdHa3GjRsrJCTEu65Tp05yu90qKytTaGhonc45kFzL7+/58+e1atUqZWRk1OUUA1pNru/UqVM1YsQI76tSXbp00fnz5zVq1Cg999xzPJH+QE2ub6tWrfTuu+/qwoULOn36tGJiYjRp0iS1bdv2eky5wYuKiqr234fdbld4eHidPS7/VVQjNDRU8fHxys3N9a6rqKhQbm6uXC7XVR2jvLxc+/btU3R0dF1NM2DV5Pr27NlTR48eVUVFhXfdZ599pujoaOLkR67l93f16tUqLS3VY489VtfTDFg1ub7/+Mc/qkRIZWxb/L1WH9fy+xsWFqYbb7xRly5d0jvvvKOHHnqorqf7s+ByuXz+fUhSTk7OVT8f1lid3oIbwFatWmXZbDYrKyvLOnjwoDVq1CgrIiLC+9HhESNGWJMmTfKOf/75563NmzdbX3zxhZWfn28NHTrUCgsLsw4cOFBfp2A0f69vQUGB1bx5c2vMmDHWkSNHrPXr11uRkZHWrFmz6usUjObv9a3Uq1cva8iQIdd7ugHH3+s7ffp0q3nz5taf//xn69ixY9aWLVusdu3aWY888kh9nYLR/L2+u3btst555x3riy++sLZv32716dPHatOmjfXdd9/V0xmY7ezZs9Ynn3xiffLJJ5Yka968edYnn3xi/d///Z9lWZY1adIka8SIEd7xlR8znjhxonXo0CFr0aJFfMy4vr388stW69atrdDQUKt79+7Wrl27vNvuv/9+KzU11fvzuHHjvGOdTqfVv39/PoN/Bf5cX8uyrJ07d1oJCQmWzWaz2rZta/3+97+3Ll26dJ1nHTj8vb6HDx+2JFlbtmy5zjMNTP5c34sXL1ozZsyw2rVrZ4WFhVmxsbHWb3/7W55Af4I/13fr1q1Wp06dLJvNZrVs2dIaMWKE9fXXX9fDrAPD3/72N0tSlaXymqamplr3339/lX26du1qhYaGWm3btr0u35EUZFm8vggAAMzCPSgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACM8/8AxhTtUc6FNfgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(xcomet_cpo_score, bins = 20)\n",
    "plt.title('XCOMET')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "44361d94-c9c9-44f9-98cd-6ba50bfc4401",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:01:19.195101Z",
     "iopub.status.busy": "2024-10-18T07:01:19.194783Z",
     "iopub.status.idle": "2024-10-18T07:01:19.213362Z",
     "shell.execute_reply": "2024-10-18T07:01:19.212779Z",
     "shell.execute_reply.started": "2024-10-18T07:01:19.195081Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9625105346321118"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cpo = 0.9626026108213093\n",
    "# modpo = 0.9377332176762843\n",
    "# alma = 0.9512363975580502\n",
    "\n",
    "import numpy as np\n",
    "np.mean(np.array(xcomet_cpo_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "0f5998d0-4cf8-471c-90a0-02df966d10f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:01:19.214865Z",
     "iopub.status.busy": "2024-10-18T07:01:19.214591Z",
     "iopub.status.idle": "2024-10-18T07:01:19.987202Z",
     "shell.execute_reply": "2024-10-18T07:01:19.986507Z",
     "shell.execute_reply.started": "2024-10-18T07:01:19.214847Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py:1142: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "import torch \n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"cointegrated/roberta-large-cola-krishna2020\")\n",
    "model_fluency = AutoModelForSequenceClassification.from_pretrained(\"cointegrated/roberta-large-cola-krishna2020\")\n",
    "model_fluency = model_fluency.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "1b277ecc-1f99-43e8-b0bf-c9ef7dce470a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:01:19.988304Z",
     "iopub.status.busy": "2024-10-18T07:01:19.987964Z",
     "iopub.status.idle": "2024-10-18T07:01:20.002238Z",
     "shell.execute_reply": "2024-10-18T07:01:20.001709Z",
     "shell.execute_reply.started": "2024-10-18T07:01:19.988277Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def fluency(data: str) -> float:\n",
    "    return torch.softmax(model_fluency(**tokenizer(data, return_tensors=\"pt\").to(device)).logits, dim = -1)[0][0].detach().cpu().float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "7752dd61-d054-43e5-b8ef-e07404922712",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:01:20.003454Z",
     "iopub.status.busy": "2024-10-18T07:01:20.002825Z",
     "iopub.status.idle": "2024-10-18T07:01:20.056123Z",
     "shell.execute_reply": "2024-10-18T07:01:20.055662Z",
     "shell.execute_reply.started": "2024-10-18T07:01:20.003424Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Now we have four-month-old mice that no longer have diabetes,\" he added.',\n",
       " 'According to the warning of Dr. Ehud Ur, a professor of medicine at Dalhousie University in Halifax (Nova Scotia) and the chair of the Clinical-Scientific Division of the Canadian Diabetes Association, the research is still in its early stages.')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_fluency = [item['mt'] for item in data]\n",
    "data_fluency[0], data_fluency[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "3b2743fd-763e-4ccd-a693-7900dcd51ca2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:01:20.057200Z",
     "iopub.status.busy": "2024-10-18T07:01:20.056749Z",
     "iopub.status.idle": "2024-10-18T07:01:40.559857Z",
     "shell.execute_reply": "2024-10-18T07:01:40.559193Z",
     "shell.execute_reply.started": "2024-10-18T07:01:20.057182Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "fluency_cpo_score = [fluency(item) for item in data_fluency]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3e35a610-f161-4be7-b51a-d569cd7de9aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:01:40.560885Z",
     "iopub.status.busy": "2024-10-18T07:01:40.560621Z",
     "iopub.status.idle": "2024-10-18T07:01:40.739645Z",
     "shell.execute_reply": "2024-10-18T07:01:40.739075Z",
     "shell.execute_reply.started": "2024-10-18T07:01:40.560866Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Fluency')"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGzCAYAAAAFROyYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAikUlEQVR4nO3de1TUdeL/8RcXuaTMoCaDFCa5lZqWF5LwkltxxKQ2T+yWKxG2rlZirVoabqmpKep2Mc10dU09adm2m13ULMPSWvGS6a6pqaWm5Q7qURi1r4Dw/v2xx/k1aeogMG/g+Thnzmk+n/d85v3pnfLsMxeCjDFGAAAAFgkO9AQAAAB+jkABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAXBJmjdvrn79+gV6GgBqGQIFwC+aP3++goKCznnLyckJ9PQA1GKhgZ4AAPuNGzdOCQkJPtvatGkToNkAqAsIFAAXdMcddygxMTHQ0wBQh/ASD4BK9cwzzygoKOis7WdeLtq3b5/P9g8++EDdunVT/fr1FRUVpbS0NG3bts1nTL9+/dSgQQP98MMP6t27txo0aKAmTZroiSeeUFlZmc/Y8vJyvfTSS2rbtq0iIiLUpEkT9ezZU1988YUkqXv37rrxxhvPOffrrrtOqampl3D2ACoLgQLggoqKinTkyBGfW2V47bXXlJaWpgYNGmjy5MkaNWqUtm/frq5du54VMmVlZUpNTVXjxo313HPPqXv37nr++ec1e/Zsn3H9+/fXkCFDFB8fr8mTJysnJ0cRERFat26dJCkzM1P/+c9/9NVXX/k8buPGjdq1a5fuv//+Sjk3AJfIAMAvmDdvnpF0ztsZV111lcnKyvLeHzNmjDnXXy1njrV3715jjDHHjx830dHRZsCAAT7j3G63cTqdPtuzsrKMJDNu3Difse3btzcdO3b03l+1apWRZB577LGznr+8vNwYY0xhYaGJiIgwTz75pM/+xx57zNSvX9+cOHHiAv9WAFQH3oMC4IJmzJiha6+9tlKPuXLlShUWFur3v/+9zxWZkJAQJSUl6ZNPPjnrMQ8//LDP/W7duum1117z3v/nP/+poKAgjRkz5qzHnnnZyel06u6779Ybb7yh3NxcBQUFqaysTG+++aZ69+6t+vXrV9YpArgEBAqAC+rUqVOlv0l29+7dkqTbbrvtnPsdDofP/TPvJ/mphg0b6tixY9773377reLi4tSoUaPzPvcDDzygN998U5999pluueUWffzxxyooKFBmZmZFTgVAFSBQAFSqc71BVtI538wq/e99KLGxsWeNDw31/espJCSkkmYopaamyuVyaeHChbrlllu0cOFCxcbGKiUlpdKeA8ClIVAAVKqGDRtKkgoLCxUdHe3d/t133/mMa9GihSQpJiam0sKgRYsW+vDDD3X06NHzXkUJCQlR3759NX/+fE2ePFnvvPOOBgwYUKkRBODS8CkeAJXqTHisWbPGu+3kyZNasGCBz7jU1FQ5HA5NnDhRpaWlZx3n8OHDfj93enq6jDEaO3bsWfuMMT73MzMzdezYMT300EM6ceIEn94BLMMVFACVqkePHmrWrJn69++v4cOHKyQkRK+++qqaNGmi/fv3e8c5HA7NnDlTmZmZ6tChg/r06eMds2zZMnXp0kUvv/yyX8996623KjMzU9OmTdPu3bvVs2dPlZeX67PPPtOtt96qwYMHe8e2b99ebdq00VtvvaVWrVqpQ4cOlfbvAMClI1AAVKp69eppyZIlGjRokEaNGqXY2FgNGTJEDRs21IMPPugztm/fvoqLi9OkSZP0l7/8RcXFxbriiivUrVu3s8ZerHnz5umGG27Q3LlzNXz4cDmdTiUmJqpz585njX3ggQc0YsQI3hwLWCjI/Py6JwDUES+99JKGDh2qffv2qVmzZoGeDoCfIFAA1EnGGN14441q3LjxOb9zBUBg8RIPgDrl5MmTeu+99/TJJ59o69atevfddwM9JQDnwBUUAHXKvn37lJCQoOjoaA0aNEgTJkwI9JQAnAOBAgAArMP3oAAAAOsQKAAAwDo18k2y5eXlOnjwoKKion7x934AAAC7GGN0/PhxxcXFKTj4/NdIamSgHDx4UPHx8YGeBgAAqIADBw7oyiuvPO+YGhkoUVFRkv53gj//lewAAMBOHo9H8fHx3p/j51MjA+XMyzoOh4NAAQCghrmYt2fwJlkAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFgnNNATAAAAFdM8Z1mVHXvfpLQqO/bF4AoKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsI5fgVJWVqZRo0YpISFBkZGRatGihcaPHy9jjHeMMUajR49W06ZNFRkZqZSUFO3evdvnOEePHlVGRoYcDoeio6PVv39/nThxonLOCAAA1Hh+BcrkyZM1c+ZMvfzyy9qxY4cmT56sKVOmaPr06d4xU6ZM0bRp0zRr1iytX79e9evXV2pqqk6dOuUdk5GRoW3btmnlypVaunSp1qxZo4EDB1beWQEAgBotyPz08scF3HnnnXK5XJo7d653W3p6uiIjI7Vw4UIZYxQXF6fHH39cTzzxhCSpqKhILpdL8+fPV58+fbRjxw61bt1aGzduVGJioiRpxYoV6tWrl77//nvFxcVdcB4ej0dOp1NFRUVyOBz+njMAALVC85xlVXbsfZPSKv2Y/vz89usKSufOnZWXl6ddu3ZJkv7973/r888/1x133CFJ2rt3r9xut1JSUryPcTqdSkpKUn5+viQpPz9f0dHR3jiRpJSUFAUHB2v9+vXnfN7i4mJ5PB6fGwAAqL1C/Rmck5Mjj8ejli1bKiQkRGVlZZowYYIyMjIkSW63W5Lkcrl8Hudyubz73G63YmJifCcRGqpGjRp5x/xcbm6uxo4d689UAQBADebXFZS///3vWrRokV5//XV9+eWXWrBggZ577jktWLCgquYnSRo5cqSKioq8twMHDlTp8wEAgMDy6wrK8OHDlZOToz59+kiS2rZtq++++065ubnKyspSbGysJKmgoEBNmzb1Pq6goEDt2rWTJMXGxurQoUM+xz19+rSOHj3qffzPhYeHKzw83J+pAgCAGsyvKyg//vijgoN9HxISEqLy8nJJUkJCgmJjY5WXl+fd7/F4tH79eiUnJ0uSkpOTVVhYqE2bNnnHrFq1SuXl5UpKSqrwiQAAgNrDrysod911lyZMmKBmzZrp+uuv1+bNm/XCCy/oD3/4gyQpKChIQ4YM0bPPPqtrrrlGCQkJGjVqlOLi4tS7d29JUqtWrdSzZ08NGDBAs2bNUmlpqQYPHqw+ffpc1Cd4AABA7edXoEyfPl2jRo3SoEGDdOjQIcXFxemhhx7S6NGjvWNGjBihkydPauDAgSosLFTXrl21YsUKRUREeMcsWrRIgwcP1u23367g4GClp6dr2rRplXdWAACgRvPre1BswfegAADA96AAAABUKwIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHX8DpQffvhB999/vxo3bqzIyEi1bdtWX3zxhXe/MUajR49W06ZNFRkZqZSUFO3evdvnGEePHlVGRoYcDoeio6PVv39/nThx4tLPBgAA1Ap+BcqxY8fUpUsX1atXTx988IG2b9+u559/Xg0bNvSOmTJliqZNm6ZZs2Zp/fr1ql+/vlJTU3Xq1CnvmIyMDG3btk0rV67U0qVLtWbNGg0cOLDyzgoAANRoQcYYc7GDc3Jy9K9//UufffbZOfcbYxQXF6fHH39cTzzxhCSpqKhILpdL8+fPV58+fbRjxw61bt1aGzduVGJioiRpxYoV6tWrl77//nvFxcVdcB4ej0dOp1NFRUVyOBwXO30AAGqV5jnLquzY+yalVfox/fn57dcVlPfee0+JiYn63e9+p5iYGLVv315z5szx7t+7d6/cbrdSUlK825xOp5KSkpSfny9Jys/PV3R0tDdOJCklJUXBwcFav379OZ+3uLhYHo/H5wYAAGovvwJlz549mjlzpq655hp9+OGHeuSRR/TYY49pwYIFkiS32y1JcrlcPo9zuVzefW63WzExMT77Q0ND1ahRI++Yn8vNzZXT6fTe4uPj/Zk2AACoYfwKlPLycnXo0EETJ05U+/btNXDgQA0YMECzZs2qqvlJkkaOHKmioiLv7cCBA1X6fAAAILD8CpSmTZuqdevWPttatWql/fv3S5JiY2MlSQUFBT5jCgoKvPtiY2N16NAhn/2nT5/W0aNHvWN+Ljw8XA6Hw+cGAABqL78CpUuXLtq5c6fPtl27dumqq66SJCUkJCg2NlZ5eXne/R6PR+vXr1dycrIkKTk5WYWFhdq0aZN3zKpVq1ReXq6kpKQKnwgAAKg9Qv0ZPHToUHXu3FkTJ07Uvffeqw0bNmj27NmaPXu2JCkoKEhDhgzRs88+q2uuuUYJCQkaNWqU4uLi1Lt3b0n/u+LSs2dP70tDpaWlGjx4sPr06XNRn+ABAAC1n1+BctNNN2nJkiUaOXKkxo0bp4SEBE2dOlUZGRneMSNGjNDJkyc1cOBAFRYWqmvXrlqxYoUiIiK8YxYtWqTBgwfr9ttvV3BwsNLT0zVt2rTKOysAAFCj+fU9KLbge1AAAOB7UAAAAKoVgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA61xSoEyaNElBQUEaMmSId9upU6eUnZ2txo0bq0GDBkpPT1dBQYHP4/bv36+0tDRddtlliomJ0fDhw3X69OlLmQoAAKhFKhwoGzdu1F//+lfdcMMNPtuHDh2q999/X2+99ZZWr16tgwcP6p577vHuLysrU1pamkpKSrR27VotWLBA8+fP1+jRoyt+FgAAoFapUKCcOHFCGRkZmjNnjho2bOjdXlRUpLlz5+qFF17Qbbfdpo4dO2revHlau3at1q1bJ0n66KOPtH37di1cuFDt2rXTHXfcofHjx2vGjBkqKSmpnLMCAAA1WoUCJTs7W2lpaUpJSfHZvmnTJpWWlvpsb9mypZo1a6b8/HxJUn5+vtq2bSuXy+Udk5qaKo/Ho23btp3z+YqLi+XxeHxuAACg9gr19wGLFy/Wl19+qY0bN561z+12KywsTNHR0T7bXS6X3G63d8xP4+TM/jP7ziU3N1djx471d6oAAKCG8usKyoEDB/SnP/1JixYtUkRERFXN6SwjR45UUVGR93bgwIFqe24AAFD9/AqUTZs26dChQ+rQoYNCQ0MVGhqq1atXa9q0aQoNDZXL5VJJSYkKCwt9HldQUKDY2FhJUmxs7Fmf6jlz/8yYnwsPD5fD4fC5AQCA2suvQLn99tu1detWbdmyxXtLTExURkaG95/r1aunvLw872N27typ/fv3Kzk5WZKUnJysrVu36tChQ94xK1eulMPhUOvWrSvptAAAQE3m13tQoqKi1KZNG59t9evXV+PGjb3b+/fvr2HDhqlRo0ZyOBx69NFHlZycrJtvvlmS1KNHD7Vu3VqZmZmaMmWK3G63nn76aWVnZys8PLySTgsAANRkfr9J9kJefPFFBQcHKz09XcXFxUpNTdUrr7zi3R8SEqKlS5fqkUceUXJysurXr6+srCyNGzeusqcCAABqqCBjjAn0JPzl8XjkdDpVVFTE+1EAAHVW85xlVXbsfZPSKv2Y/vz85nfxAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKzjV6Dk5ubqpptuUlRUlGJiYtS7d2/t3LnTZ8ypU6eUnZ2txo0bq0GDBkpPT1dBQYHPmP379ystLU2XXXaZYmJiNHz4cJ0+ffrSzwYAANQKfgXK6tWrlZ2drXXr1mnlypUqLS1Vjx49dPLkSe+YoUOH6v3339dbb72l1atX6+DBg7rnnnu8+8vKypSWlqaSkhKtXbtWCxYs0Pz58zV69OjKOysAAFCjBRljTEUffPjwYcXExGj16tW65ZZbVFRUpCZNmuj111/Xb3/7W0nS119/rVatWik/P18333yzPvjgA9155506ePCgXC6XJGnWrFl68skndfjwYYWFhV3weT0ej5xOp4qKiuRwOCo6fQAAarTmOcuq7Nj7JqVV+jH9+fl9Se9BKSoqkiQ1atRIkrRp0yaVlpYqJSXFO6Zly5Zq1qyZ8vPzJUn5+flq27atN04kKTU1VR6PR9u2bTvn8xQXF8vj8fjcAABA7VXhQCkvL9eQIUPUpUsXtWnTRpLkdrsVFham6Ohon7Eul0tut9s75qdxcmb/mX3nkpubK6fT6b3Fx8dXdNoAAKAGqHCgZGdn66uvvtLixYsrcz7nNHLkSBUVFXlvBw4cqPLnBAAAgRNakQcNHjxYS5cu1Zo1a3TllVd6t8fGxqqkpESFhYU+V1EKCgoUGxvrHbNhwwaf4535lM+ZMT8XHh6u8PDwikwVAADUQH5dQTHGaPDgwVqyZIlWrVqlhIQEn/0dO3ZUvXr1lJeX5922c+dO7d+/X8nJyZKk5ORkbd26VYcOHfKOWblypRwOh1q3bn0p5wIAAGoJv66gZGdn6/XXX9e7776rqKgo73tGnE6nIiMj5XQ61b9/fw0bNkyNGjWSw+HQo48+quTkZN18882SpB49eqh169bKzMzUlClT5Ha79fTTTys7O5urJAAAQJKfgTJz5kxJ0q9//Wuf7fPmzVO/fv0kSS+++KKCg4OVnp6u4uJipaam6pVXXvGODQkJ0dKlS/XII48oOTlZ9evXV1ZWlsaNG3dpZwIAAGqNS/oelEDhe1AAAOB7UAAAAKoVgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArBMa6AkAAFCbNc9ZFugp1EhcQQEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYhy9qAwDUGHzpWd3BFRQAAGAdAgUAAFiHQAEAANYJaKDMmDFDzZs3V0REhJKSkrRhw4ZATgcAAFgiYG+SffPNNzVs2DDNmjVLSUlJmjp1qlJTU7Vz507FxMQEaloAUGdU5RtO901Kq7Jjo24IMsaYQDxxUlKSbrrpJr388suSpPLycsXHx+vRRx9VTk7OeR/r8XjkdDpVVFQkh8NRHdMFgIDhkysIhKqITH9+fgfkCkpJSYk2bdqkkSNHercFBwcrJSVF+fn5Z40vLi5WcXGx935RUZGk/50oUBu0GfNhlR37q7GpVXZs/H9VuYZAIFTFz9gzx7yYayMBCZQjR46orKxMLpfLZ7vL5dLXX3991vjc3FyNHTv2rO3x8fFVNkegtnBODfQMANREVfl3x/Hjx+V0Os87pkZ8UdvIkSM1bNgw7/3y8nIdPXpUjRs3VlBQUABnVvU8Ho/i4+N14MABXs6yEOtjN9bHbqyP3apifYwxOn78uOLi4i44NiCBcvnllyskJEQFBQU+2wsKChQbG3vW+PDwcIWHh/tsi46OrsopWsfhcPAH2GKsj91YH7uxPnar7PW50JWTMwLyMeOwsDB17NhReXl53m3l5eXKy8tTcnJyIKYEAAAsErCXeIYNG6asrCwlJiaqU6dOmjp1qk6ePKkHH3wwUFMCAACWCFig3HfffTp8+LBGjx4tt9utdu3aacWKFWe9cbauCw8P15gxY856iQt2YH3sxvrYjfWxW6DXJ2DfgwIAAPBL+F08AADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BIoFZsyYoebNmysiIkJJSUnasGHDL46dM2eOunXrpoYNG6phw4ZKSUk573hcOn/W56cWL16soKAg9e7du2onWMf5uz6FhYXKzs5W06ZNFR4ermuvvVbLly+vptnWPf6uz9SpU3XdddcpMjJS8fHxGjp0qE6dOlVNs61b1qxZo7vuuktxcXEKCgrSO++8c8HHfPrpp+rQoYPCw8P1q1/9SvPnz6+6CRoE1OLFi01YWJh59dVXzbZt28yAAQNMdHS0KSgoOOf4vn37mhkzZpjNmzebHTt2mH79+hmn02m+//77ap553eDv+pyxd+9ec8UVV5hu3bqZu+++u3omWwf5uz7FxcUmMTHR9OrVy3z++edm79695tNPPzVbtmyp5pnXDf6uz6JFi0x4eLhZtGiR2bt3r/nwww9N06ZNzdChQ6t55nXD8uXLzVNPPWXefvttI8ksWbLkvOP37NljLrvsMjNs2DCzfft2M336dBMSEmJWrFhRJfMjUAKsU6dOJjs723u/rKzMxMXFmdzc3It6/OnTp01UVJRZsGBBVU2xTqvI+pw+fdp07tzZ/O1vfzNZWVkEShXyd31mzpxprr76alNSUlJdU6zT/F2f7Oxsc9ttt/lsGzZsmOnSpUuVzhPmogJlxIgR5vrrr/fZdt9995nU1NQqmRMv8QRQSUmJNm3apJSUFO+24OBgpaSkKD8//6KO8eOPP6q0tFSNGjWqqmnWWRVdn3HjxikmJkb9+/evjmnWWRVZn/fee0/JycnKzs6Wy+VSmzZtNHHiRJWVlVXXtOuMiqxP586dtWnTJu/LQHv27NHy5cvVq1evapkzzi8/P99nPSUpNTX1on9e+StgX3UP6ciRIyorKzvr6/1dLpe+/vrrizrGk08+qbi4uLP+o8Glq8j6fP7555o7d662bNlSDTOs2yqyPnv27NGqVauUkZGh5cuX65tvvtGgQYNUWlqqMWPGVMe064yKrE/fvn115MgRde3aVcYYnT59Wg8//LD+/Oc/V8eUcQFut/uc6+nxePR///d/ioyMrNTn4wpKDTZp0iQtXrxYS5YsUURERKCnU+cdP35cmZmZmjNnji6//PJATwfnUF5erpiYGM2ePVsdO3bUfffdp6eeekqzZs0K9NSg/70Bc+LEiXrllVf05Zdf6u2339ayZcs0fvz4QE8NAcAVlAC6/PLLFRISooKCAp/tBQUFio2NPe9jn3vuOU2aNEkff/yxbrjhhqqcZp3l7/p8++232rdvn+666y7vtvLycklSaGiodu7cqRYtWlTtpOuQivz5adq0qerVq6eQkBDvtlatWsntdqukpERhYWFVOue6pCLrM2rUKGVmZuqPf/yjJKlt27Y6efKkBg4cqKeeekrBwfw/dSDFxsaecz0dDkelXz2RuIISUGFhYerYsaPy8vK828rLy5WXl6fk5ORffNyUKVM0fvx4rVixQomJidUx1TrJ3/Vp2bKltm7dqi1btnhvv/nNb3Trrbdqy5Ytio+Pr87p13oV+fPTpUsXffPNN95wlKRdu3apadOmxEklq8j6/Pjjj2dFyJmYNPxe24BLTk72WU9JWrly5Xl/Xl2SKnnrLS7a4sWLTXh4uJk/f77Zvn27GThwoImOjjZut9sYY0xmZqbJycnxjp80aZIJCwsz//jHP8x///tf7+348eOBOoVazd/1+Tk+xVO1/F2f/fv3m6ioKDN48GCzc+dOs3TpUhMTE2OeffbZQJ1Crebv+owZM8ZERUWZN954w+zZs8d89NFHpkWLFubee+8N1CnUasePHzebN282mzdvNpLMCy+8YDZv3my+++47Y4wxOTk5JjMz0zv+zMeMhw8fbnbs2GFmzJjBx4xru+nTp5tmzZqZsLAw06lTJ7Nu3Trvvu7du5usrCzv/auuuspIOus2ZsyY6p94HeHP+vwcgVL1/F2ftWvXmqSkJBMeHm6uvvpqM2HCBHP69OlqnnXd4c/6lJaWmmeeeca0aNHCREREmPj4eDNo0CBz7Nix6p94HfDJJ5+c8+fJmTXJysoy3bt3P+sx7dq1M2FhYebqq6828+bNq7L5BRnDdTMAAGAX3oMCAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOv8PnQuOaLwVhRcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.hist(fluency_cpo_score, bins = 20)\n",
    "plt.title('Fluency')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8a30b4b2-40ab-4ddd-81d0-1cee0d6462ea",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:01:40.740686Z",
     "iopub.status.busy": "2024-10-18T07:01:40.740398Z",
     "iopub.status.idle": "2024-10-18T07:01:40.807063Z",
     "shell.execute_reply": "2024-10-18T07:01:40.806533Z",
     "shell.execute_reply.started": "2024-10-18T07:01:40.740666Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.972777"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cpo = 0.9737626\n",
    "# modpo = 0.93584627\n",
    "# alma = 0.9457284 (??? 0.037422378 ???)\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "np.mean(np.array(fluency_cpo_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff4449be-c2bc-46ad-885c-2679945fdafc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:16:27.812285Z",
     "iopub.status.busy": "2024-10-18T07:16:27.811679Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching 6 files:   0%|          | 0/6 [00:00<?, ?it/s]/home/jupyter/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py:983: UserWarning: Not enough free disk space to download the file. The expected file size is: 42868.10 MB. The target location /tmp/xdg_cache/huggingface/hub/models--Unbabel--wmt23-cometkiwi-da-xxl/blobs only has 35737.21 MB free disk space.\n",
      "  warnings.warn(\n",
      "/home/jupyter/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py:983: UserWarning: Not enough free disk space to download the file. The expected file size is: 42868.10 MB. The target location /tmp/xdg_cache/huggingface/hub/models--Unbabel--wmt23-cometkiwi-da-xxl/blobs only has 35737.21 MB free disk space.\n",
      "  warnings.warn(\n",
      "Fetching 6 files:  50%|█████     | 3/6 [04:35<04:35, 91.77s/it]\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "\"Model 'Unbabel/wmt23-cometkiwi-da-xxl' not supported by COMET.\"",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/comet/models/__init__.py\u001b[0m in \u001b[0;36mdownload_model\u001b[0;34m(model, saving_directory, local_files_only)\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m         model_path = snapshot_download(\n\u001b[0m\u001b[1;32m     49\u001b[0m             \u001b[0mrepo_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msaving_directory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_files_only\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlocal_files_only\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py\u001b[0m in \u001b[0;36m_inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/_snapshot_download.py\u001b[0m in \u001b[0;36msnapshot_download\u001b[0;34m(repo_id, repo_type, revision, cache_dir, local_dir, library_name, library_version, user_agent, proxies, etag_timeout, force_download, token, local_files_only, allow_patterns, ignore_patterns, max_workers, tqdm_class, headers, endpoint, local_dir_use_symlinks, resume_download)\u001b[0m\n\u001b[1;32m    292\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 293\u001b[0;31m         thread_map(\n\u001b[0m\u001b[1;32m    294\u001b[0m             \u001b[0m_inner_hf_hub_download\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/tqdm/contrib/concurrent.py\u001b[0m in \u001b[0;36mthread_map\u001b[0;34m(fn, *iterables, **tqdm_kwargs)\u001b[0m\n\u001b[1;32m     68\u001b[0m     \u001b[0;32mfrom\u001b[0m \u001b[0mconcurrent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfutures\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mThreadPoolExecutor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_executor_map\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mThreadPoolExecutor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0miterables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mtqdm_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/tqdm/contrib/concurrent.py\u001b[0m in \u001b[0;36m_executor_map\u001b[0;34m(PoolExecutor, fn, *iterables, **tqdm_kwargs)\u001b[0m\n\u001b[1;32m     50\u001b[0m                           initargs=(lk,)) as ex:\n\u001b[0;32m---> 51\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0miterables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1181\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1182\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1183\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult_iterator\u001b[0;34m()\u001b[0m\n\u001b[1;32m    620\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 621\u001b[0;31m                         \u001b[0;32myield\u001b[0m \u001b[0m_result_or_cancel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    622\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m_result_or_cancel\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    318\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfut\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    457\u001b[0m                 \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mFINISHED\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 458\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    459\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    402\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 403\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    404\u001b[0m             \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/thread.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/_snapshot_download.py\u001b[0m in \u001b[0;36m_inner_hf_hub_download\u001b[0;34m(repo_file)\u001b[0m\n\u001b[1;32m    266\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_inner_hf_hub_download\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepo_file\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 267\u001b[0;31m         return hf_hub_download(\n\u001b[0m\u001b[1;32m    268\u001b[0m             \u001b[0mrepo_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/utils/_deprecation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mwarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mFutureWarning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py\u001b[0m in \u001b[0;36m_inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36mhf_hub_download\u001b[0;34m(repo_id, filename, subfolder, repo_type, revision, library_name, library_version, cache_dir, local_dir, user_agent, force_download, proxies, etag_timeout, token, local_files_only, headers, endpoint, legacy_cache_layout, resume_download, force_filename, local_dir_use_symlinks)\u001b[0m\n\u001b[1;32m   1231\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1232\u001b[0;31m         return _hf_hub_download_to_cache_dir(\n\u001b[0m\u001b[1;32m   1233\u001b[0m             \u001b[0;31m# Destination\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36m_hf_hub_download_to_cache_dir\u001b[0;34m(cache_dir, repo_id, filename, repo_type, revision, endpoint, etag_timeout, headers, proxies, token, local_files_only, force_download)\u001b[0m\n\u001b[1;32m   1380\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mWeakFileLock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlock_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1381\u001b[0;31m         _download_to_tmp_and_move(\n\u001b[0m\u001b[1;32m   1382\u001b[0m             \u001b[0mincomplete_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mPath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblob_path\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\".incomplete\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36m_download_to_tmp_and_move\u001b[0;34m(incomplete_path, destination_path, url_to_download, proxies, headers, expected_size, filename, force_download)\u001b[0m\n\u001b[1;32m   1914\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1915\u001b[0;31m         http_get(\n\u001b[0m\u001b[1;32m   1916\u001b[0m             \u001b[0murl_to_download\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36mhttp_get\u001b[0;34m(url, temp_file, proxies, resume_size, headers, expected_size, displayed_filename, _nb_retries, _tqdm_bar)\u001b[0m\n\u001b[1;32m    569\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mexpected_size\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mexpected_size\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mtemp_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 570\u001b[0;31m         raise EnvironmentError(\n\u001b[0m\u001b[1;32m    571\u001b[0m             consistency_error_message.format(\n",
      "\u001b[0;31mOSError\u001b[0m: Consistency check failed: file should be of size 42868098631 but has size 29350625529 (model.ckpt).\nWe are sorry for the inconvenience. Please retry with `force_download=True`.\nIf the issue persists, please let us know by opening an issue on https://github.com/huggingface/huggingface_hub.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/comet/models/__init__.py\u001b[0m in \u001b[0;36mdownload_model\u001b[0;34m(model, saving_directory, local_files_only)\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 53\u001b[0;31m             \u001b[0mcheckpoint_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdownload_model_legacy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaving_directory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     54\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/comet/models/download_utils.py\u001b[0m in \u001b[0;36mdownload_model_legacy\u001b[0;34m(model, saving_directory)\u001b[0m\n\u001b[1;32m    223\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mavailable_legacy_metrics\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 224\u001b[0;31m         raise Exception(\n\u001b[0m\u001b[1;32m    225\u001b[0m             \u001b[0;34mf\"{model} is not in the `available_legacy_metrics` or is a valid checkpoint folder.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mException\u001b[0m: Unbabel/wmt23-cometkiwi-da-xxl is not in the `available_legacy_metrics` or is a valid checkpoint folder.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_4373/897186651.py\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mcomet\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdownload_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_from_checkpoint\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mmodel_path_kiwi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdownload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Unbabel/wmt23-cometkiwi-da-xxl\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mmodel_kiwi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_from_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_path_kiwi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mmodel_kiwi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_kiwi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/comet/models/__init__.py\u001b[0m in \u001b[0;36mdownload_model\u001b[0;34m(model, saving_directory, local_files_only)\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mcheckpoint_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdownload_model_legacy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaving_directory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Model '{model}' not supported by COMET.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0mcheckpoint_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"checkpoints\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"model.ckpt\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: \"Model 'Unbabel/wmt23-cometkiwi-da-xxl' not supported by COMET.\""
     ]
    }
   ],
   "source": [
    "from comet import download_model, load_from_checkpoint\n",
    "\n",
    "model_path_kiwi = download_model(\"Unbabel/wmt23-cometkiwi-da-xxl\")\n",
    "model_kiwi = load_from_checkpoint(model_path_kiwi)\n",
    "model_kiwi = model_kiwi.to(device)\n",
    "\n",
    "def kiwi(data: List[Dict[str, str]]) -> List[float]:\n",
    "    \n",
    "    '''Format\n",
    "    data = [\n",
    "    {\n",
    "        # source original text, ru\n",
    "        \"src\": \"В понедельник\",\n",
    "        \n",
    "        # machine translation, en\n",
    "        \"mt\": \"On Monday\"\n",
    "    }'''\n",
    "    \n",
    "    \n",
    "    kiwi_metric = model_kiwi.predict(data, batch_size=8, gpus=1)\n",
    "    return kiwi_metric\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3c3e1df-966c-4999-aa16-b42958214365",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_kiwi = [(item['src'], item['mt']) for item in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b0d003f-9ed8-4b79-9a47-b6c561ce7c6d",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:13:08.034521Z",
     "iopub.status.idle": "2024-10-17T23:13:08.034832Z",
     "shell.execute_reply": "2024-10-17T23:13:08.034706Z",
     "shell.execute_reply.started": "2024-10-17T23:13:08.034691Z"
    }
   },
   "outputs": [],
   "source": [
    "kiwi_cpo_score = kiwi(data_kiwi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99d99685-04a3-457c-b159-b8b5788c4f48",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DataSphere Kernel",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}