T1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7afe7687-bba4-42c6-836b-0ec5e50f00b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:59:42.008832Z",
     "iopub.status.busy": "2024-10-18T06:59:42.008463Z",
     "iopub.status.idle": "2024-10-18T06:59:42.250328Z",
     "shell.execute_reply": "2024-10-18T06:59:42.249664Z",
     "shell.execute_reply.started": "2024-10-18T06:59:42.008811Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "309c0e7df60847a6acb26c24e034d33a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4372adf4-58d2-4998-b980-5731c67cc82b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:59:53.983290Z",
     "iopub.status.busy": "2024-10-18T06:59:53.982584Z",
     "iopub.status.idle": "2024-10-18T07:00:06.469001Z",
     "shell.execute_reply": "2024-10-18T07:00:06.467976Z",
     "shell.execute_reply.started": "2024-10-18T06:59:53.983267Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "transformers 4.38.2 requires tokenizers<0.19,>=0.14, but you have tokenizers 0.20.1 which is incompatible.\n",
      "unbabel-comet 2.2.2 requires protobuf<5.0.0,>=4.24.4, but you have protobuf 3.20.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "unbabel-comet 2.2.2 requires protobuf<5.0.0,>=4.24.4, but you have protobuf 3.20.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%pip install --upgrade torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 -q\n",
    "%pip install -U tokenizers -q\n",
    "%pip install peft transformers protobuf==3.20 bitsandbytes sentencepiece sacrebleu ipython datasets evaluate deepspeed einops wandb zstandard accelerate jsonlines trl tokenizers -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f93de812-1d46-4db7-b762-104f33de9690",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:43:18.746335Z",
     "iopub.status.busy": "2024-10-18T06:43:18.745999Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/.local/lib/python3.10/site-packages/transformers/utils/hub.py:124: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "from peft import PeftModel, PeftConfig\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from comet import download_model, load_from_checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a116327-1c50-4d3d-aa02-a4dfb30f9e0a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('data/translate_cpo_kiwi.json', 'r') as f:\n",
    "    data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6755a841-ae10-4aa7-a04f-b95cb657909e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T23:19:04.310819Z",
     "iopub.status.busy": "2024-10-17T23:19:04.310509Z",
     "iopub.status.idle": "2024-10-17T23:19:04.319231Z",
     "shell.execute_reply": "2024-10-17T23:19:04.318599Z",
     "shell.execute_reply.started": "2024-10-17T23:19:04.310800Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def adequacy(en_tr, en_ref):\n",
    "    return len(en_tr) / len(en_ref)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f5a08dcd-009e-49b1-85f2-dd295fa53611",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T23:19:04.320235Z",
     "iopub.status.busy": "2024-10-17T23:19:04.319940Z",
     "iopub.status.idle": "2024-10-17T23:19:04.337403Z",
     "shell.execute_reply": "2024-10-17T23:19:04.336784Z",
     "shell.execute_reply.started": "2024-10-17T23:19:04.320215Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "adequacy_score = [adequacy(item['mt'], item['ref']) for item in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b2d7563f-b7db-45c0-bf14-4b2b8219a8e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T23:19:04.338605Z",
     "iopub.status.busy": "2024-10-17T23:19:04.338102Z",
     "iopub.status.idle": "2024-10-17T23:19:04.551431Z",
     "shell.execute_reply": "2024-10-17T23:19:04.550766Z",
     "shell.execute_reply.started": "2024-10-17T23:19:04.338584Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Отношение длины перевода на EN к референсу EN')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGzCAYAAAAFROyYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8vElEQVR4nO3deVxU9eL/8fcgMqAChoqA4UZumVt2Jcs1NcQty3stt6zMpSxv+miR0pSWi6a3vJWVdk27uWaardd9rdDrei1v4UqWCm4BigkCn98f/Ziv4wzIKCMHfD0fj/N4MJ/zOed8zmfOzLz5nHNmbMYYIwAAAAvxKekGAAAAXIqAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAchl79uzRwIEDVaNGDdntdkVERGjAgAHas2dPSTcNQBmQnJwsm81W4DRp0iRH3Q4dOshms6lnz54Frmfq1KnXsvle8cknn8hms2n9+vUl3ZRSIf+4cDc1bNjQUW/OnDmy2Wzy9/fXkSNH3K7nlltuuZZNL5RvSTfAypYuXap+/fopJCREQ4YMUZ06dZScnKxZs2bpk08+0cKFC3XvvfeWdDMBlAH9+vVTt27dXMpbtGjhUvbll19q+/btatmy5bVoGkqBG2+8UQkJCS7lwcHBLmVZWVmaNGmS3nrrrWvRtCtGQCnAgQMHNGjQINWtW1cbN25UtWrVHPP++te/qm3btho0aJB2796tunXrlmBLAZQFt956qwYOHHjZejVr1tSZM2cUHx+vzz///Bq0DKVBcHBwkY4fSWrevLnef/99xcXFKSIiwsstu3Kc4inAlClTdO7cOc2cOdMpnEhS1apVNWPGDGVmZuq1116TJE2cOLHQYdqLhysLGkabOnWqbDabkpOTncrfeecdNW7c2HGKaeTIkUpLS3Oq426d7ta3f/9+2Ww2vf322051f/rpJ/35z39WSEiI/P39ddttt7m8+eUPD27bts2p/OTJk7LZbJo4caKjLL8/Lnb27FmFhYW5HbrdsmWLunbtquDgYFWoUEHt27fXt99+69JHBSmo/zt06OBSt7Ah9UvrzJkzx2nZkSNHymaz6aGHHrqqfjl58mSB+1K7dm2n9btzudMCl+53VlaWJkyYoJtuukl2u12RkZF69tlnlZWV5VTPZrPpiSee0Lx589SgQQP5+/urZcuW2rhxo0sbjhw5okceeUTVq1eX3W5X48aN9cEHHzjVWb9+vVO77Ha76tevr4SEBF36Q+o7d+5UbGysgoKCVKlSJXXq1EmbN292u/8PPfSQ2/2+tN8+++wzde/eXREREbLb7YqKitLLL7+s3NzcQvtXKvi52rZtm8uxsXv3bj300EOqW7eu/P39FRYWpkceeUSnTp267HY8FRgYqNGjR+uLL77Qjh07PF7e3bF95swZtWzZUnXq1NGxY8cKXb527drq0aOHVq5cqebNm8vf318333yzli5d6lI3LS1NTz31lCIjI2W323XTTTdp8uTJysvLc6p38OBBtW/fXgEBAbr77rsd/XbkyBG1adNGAQEB6tGjh06cOOGyjblz56ply5YKCAhQSEiIHnjgAf3yyy9OdfLfH7dv36477rhDAQEBqlOnjt577z2X9Xn6WrlUjx49VLt2baeyvLw8/eMf/1CTJk3k7++vatWqqWvXro73jPbt26tZs2Zueltq0KCBYmJi3M67Us8//7xyc3OdTh9aESMoBfjiiy9Uu3ZttW3b1u38du3aqXbt2vrqq68kSffdd59uuukmx/zRo0erUaNGGjZsmKOsUaNGHrdj4sSJio+PV+fOnfXYY48pKSlJ7777rrZu3apvv/1W5cuX93idl9qzZ4/uvPNO1ahRQ2PHjlXFihX18ccfq3fv3lqyZEmxncb6+9//rtTUVJfytWvXKjY2Vi1bttSECRPk4+Oj2bNn66677tKmTZvUqlWrIm/j3XffVaVKlSRJcXFxhdYdNmyY4/ldunSpPv3000Lr79+/X++//36R23ItuDstcOl+5+XlqVevXvrmm280bNgwNWrUSN9//73eeOMN7d27V8uWLXOqv2HDBi1atEijRo2S3W7XO++8o65du+o///mPIwSnpqbq9ttvd7xJV6tWTf/+9781ZMgQZWRk6KmnnnJa5/PPP69GjRrp999/16JFi/T8888rNDRUQ4YMkfTHMdi2bVsFBQXp2WefVfny5TVjxgx16NBBGzZsUHR0tMu+2+12/fOf/3Q8fvTRR13qzJkzR5UqVdKYMWNUqVIlrV27Vi+++KIyMjI0ZcqUIvfz5axatUoHDx7Uww8/rLCwMO3Zs0czZ87Unj17tHnzZpew7s65c+fcBtfKlSvL19f5rfqvf/2r3njjDU2cOPGqR1EuXLigPn366PDhw/r2228VHh5+2WX27dun+++/XyNGjNDgwYM1e/Zs/eUvf9Hy5cvVpUsXx/60b99eR44c0fDhw1WzZk199913iouL07FjxzRt2jRJfxyfPXr00LFjx/Tiiy8qNTVV48ePlyQ9/fTTGjJkiDp27KgpU6Zo4MCBWrFihaMdr776qsaPH6++ffvq0Ucf1YkTJ/TWW2+pXbt22rlzpypXruyo+9tvv6lbt27q27ev+vXrp48//liPPfaY/Pz89Mgjjzja4slrpaiGDBmiOXPmKDY2Vo8++qhycnK0adMmbd68WbfddpsGDRqkoUOH6ocffnD6R3Pr1q3au3evxo0bd9lt5Obmuj1+AgICVLFiRaeyOnXq6MEHH9T777+vsWPHWncUxcBFWlqakWTuueeeQuv16tXLSDIZGRku82rVqmUGDx7sdrn27dubxo0bu5RPmTLFSDKHDh0yxhhz/Phx4+fnZ+6++26Tm5vrqPf2228bSeaDDz4odJ2Xrs8YY/bt22ckmbfeestR1qlTJ9OkSRNz/vx5R1leXp654447TL169Rxls2fPNpLM1q1bnbZz4sQJI8lMmDDBUTZhwgRz8eF1/PhxExgYaGJjY40ks27dOsd26tWrZ2JiYkxeXp6j/rlz50ydOnVMly5d3HWhi+eff95IMidPnnSUNW7c2LRv396lbn4ffPjhhwW299ChQ0aSmT17tqOsb9++5pZbbjGRkZFOz+2V9MuJEycK3JfCjp1L2zdlyhSXeZfu90cffWR8fHzMpk2bnOq99957RpL59ttvHWWSjCSzbds2R9nPP/9s/P39zb333usoGzJkiAkPD3fqb2OMeeCBB0xwcLA5d+6cMcaYdevWOT3fxhhz/vx54+PjYx5//HFHWe/evY2fn585cOCAo+zo0aMmMDDQtGvXzmUf+/fvbypVquRUVrFiRZd+y2/HxYYPH24qVKjgdLy7U9BztXXrVpdjw912FixYYCSZjRs3Frqd/OeyoCkxMdFR9+LXeXx8vJFktm/f7rQed8eEu+3Nnj3b5OXlmQEDBpgKFSqYLVu2FLpcvlq1ahlJZsmSJY6y9PR0Ex4eblq0aOEoe/nll03FihXN3r17nZYfO3asKVeunDl8+LAxxpilS5caSWbFihWOOk8//bSRZCZPnuwomzFjhpFkduzYYYwxJjk52ZQrV868+uqrTuv//vvvja+vr1N5+/btjSTz97//3VGWlZVlmjdvbkJDQ012drYxxvPXysiRI136p3v37qZWrVqOx2vXrjWSzKhRo1zq5r/npaWlGX9/f/Pcc885zR81apSpWLGiOXv2rMuyF8vfP3fT8OHDHfUufq86cOCA8fX1dWpXQZ9NJYVTPG6cOXNG0h9DqYXJn5+RkeHxNvLT7sXTuXPnnOqsXr1a2dnZeuqpp+Tj839P1dChQxUUFOQYvbkap0+f1tq1a9W3b1+dOXPG0ZZTp04pJiZG+/btc7naOz093andp0+fvux2Xn75ZQUHB2vUqFFO5bt27dK+ffvUv39/nTp1yrHOzMxMderUSRs3bnQZDnbn/PnzkiR/f//L1s3Ozpb0x3/gRbV9+3YtXrxYCQkJTs/FxTzpl9OnTzv209sWL16sRo0aqWHDhk7tu+uuuyRJ69atc6rfunVrp4sva9asqXvuuUcrVqxQbm6ujDFasmSJevbsKWOM0zpjYmKUnp7ucuohv28OHz6s1157TXl5eY7t5+bmauXKlerdu7fT9Vzh4eHq37+/vvnmG5fX2Pnz54v0XAcEBDj+zj++27Ztq3Pnzumnn34qUv/lP1f5U3p6eqHbOX/+vE6ePKnbb79dkop8GmbYsGFatWqVy3TzzTe7rf/Xv/5VN9xwg+Lj44u0fneeeeYZzZs3Tx9//LFHI5URERFOI6tBQUF68MEHtXPnTqWkpEj647hr27atbrjhBqf+69y5s3Jzcx2nDdesWaPAwEDHyIskx4jZxW267777JP0x4ir9MeqZl5envn37Oq0/LCxM9erVczmufX19NXz4cMdjPz8/DR8+XMePH9f27dsdbfbktZL/XF88XbhwwanOkiVLZLPZNGHCBJd+zB9ZCw4O1j333KMFCxY4Tn3m5uZq0aJF6t27t8sIiDu1a9d2e/xcOpqZr27duho0aJBmzpx52dN6JYVTPG7kB4/8oFKQogYZd3766SeXa1su9fPPP0v64xzkxfz8/FS3bl3H/Kuxf/9+GWM0fvx4x7DqpY4fP64aNWo4Hnfu3NmjbRw6dEgzZszQu+++6/Khsm/fPknS4MGDC1w+PT1dN9xwQ6HbOHnypMqXL68KFSpctj351+/knwoqirFjx6pt27bq0aOH2/POkmf9cvFzGhoaqqFDhyo+Pl7lypUr8jqKat++ffrxxx8LPN6OHz/u9LhevXouderXr69z587pxIkT8vHxUVpammbOnKmZM2cWaZ29e/d2/O3j46Nx48apT58+kqQTJ07o3LlzLse59Mdp0by8PP3yyy9q3Lixo/zkyZNu70641J49ezRu3DitXbvWJeS4CxruuGvXpU6fPq34+HgtXLjQZd+Lup169ep5dAwFBwfrqaee0oQJE7Rz587LvkYuNWPGDMc1Pr/99ptHy950000up63q168v6Y9rXMLCwrRv3z7t3r37ssfdL7/8ovDw8MueBqtatar8/Pwc15fs27dPxhi3x6skl9PfERERLh/0F7f59ttv9/i1MmvWLM2aNculXq1atRx/HzhwQBEREQoJCSl0/x588EEtWrRImzZtUrt27bR69WqlpqZq0KBBhS6Xr2LFih6/N48bN04fffSRJk2apH/84x8eLXstEFDcCA4OVnh4uHbv3l1ovd27d6tGjRoKCgryeBu1a9d2uZ5h8eLFBb7he0v+6MTTTz9d4IVYF19bI0nTp093vLClP0aQ8j9s3HnhhRdUr149DR48WJs2bXK7/SlTpqh58+Zuly9KkEhOTlbNmjWLdK4//z+8sLCwy9aVpJUrV2r16tVKTEwstJ4n/bJkyRIFBQXp3Llz+vTTT/Xqq686rr8obnl5eWrSpIlef/11t/MjIyM9Xp8kDRw4sMBg2bRpU6fHU6dOVbNmzXThwgVt3bpVr7zyinx9fd3+V1kUycnJBX4w5UtLS1P79u0VFBSkl156SVFRUfL399eOHTv03HPPFWlkTvq/5yrf3r17NXLkSKc6ffv21XfffadnnnlGzZs3V6VKlZSXl6euXbsWeTtXIv9alPj4eMc1HUW1efNmvfrqq9q6datGjx6trl27qmrVqsXWtry8PHXp0qXAYzr/tZI/+llUv//+u2P9NptN//73v90Ge0/+Acnn6WvlnnvucfmHZdy4cY73GE/ExMSoevXqmjt3rtq1a6e5c+cqLCzM49Dhibp162rgwIGaOXOmxo4d67XtXCkCSgF69Oih999/X998843atGnjMn/Tpk1KTk52GjL0hLu0u2vXLqfH+Sk8KSnJaeg7Oztbhw4dKpYDN3+95cuXL/L6WrVqpdtuu83xuLA7Unbu3KmFCxdq2bJlbt9EoqKiJP0xRHyl+5OTk6P//ve/6tq1a5Hq/+9//5PNZivSf8bGGI0dO1b33nuvY8i+IJ70S7t27RwfBr169dK3336r5cuXeyWgREVF6b///a86depUpACXP6p1sb1796pChQqO/ywDAwOVm5tb5OesZcuWjjuLYmNjdeTIEU2ePFnjx49XtWrVVKFCBSUlJbks99NPP8nHx8fpg+HEiRM6fPiw+vXrV+g2169fr1OnTmnp0qVq166do/zQoUNFanO+i58rSU4XXkp/jD6sWbNG8fHxevHFFx3l7vqxuOWPokycOLHQUUh3HnnkET3//PM6evSobr75Zo0ePVofffRRkZbNH3m9+Hjau3evJDnuYImKitLZs2cve4yEh4crMTHRZX2XOnnypLKzsx0XdEZFRckYozp16jj9Y1CQo0ePKjMz02kUxV2bPXmt3HjjjS77N23aNKeAEhUVpRUrVuj06dOFjqKUK1dO/fv315w5czR58mQtW7ZMQ4cO9cqo6sXGjRunuXPnavLkyV7dzpXgGpQCPPPMMwoICNDw4cNdbhU8ffq0RowYoQoVKuiZZ57xWhs6d+4sPz8/vfnmm063ZM6aNUvp6enq3r37VW8jNDRUHTp00IwZM9yeh3R3W58nxo4dqzvvvFO9evVyO79ly5aKiorS1KlTdfbs2Sva/sqVK5Wenq577rnnsnVzcnK0ZMkStWrVqkj/YS1cuFC7d+92+wVIxcUYI2OM196I+vbtqyNHjri9A+n33393uQ4mMTHR6bqJX375RZ999pnuvvtulStXTuXKlVOfPn20ZMkS/fDDDy7rLMpz9vvvvysnJ0c5OTkqV66c7r77bn322WdOt8SnpqZq/vz5atOmjdMIxuLFiyXpss93fn9e/NrJzs7WO++8c9n2ecLddiR5PKJxpZ566ilVrlxZL730kkfL5d/BFhERocmTJ2vu3LlauXJlkZY9evSo011vGRkZ+te//qXmzZs7Rib79u2rxMREp7tu8qWlpSknJ0fSHwHwzJkzWrNmjWP+li1bJEn/+c9/HGX5d9Dkh8377rtP5cqVU3x8vEvfG2Nc3rdzcnI0Y8YMx+Ps7GzNmDFD1apVc1xz5elrpSj69OkjY4zba4UubfegQYP022+/afjw4Tp79myRv9fkakRFRWngwIGaMWPGFY38eBMjKAWoV6+ePvzwQw0YMEBNmjRx+SbZkydPasGCBY4RAG+oVq2a4uLiFB8fr65du6pXr15KSkrSO++8oz/96U8uB+/Zs2e1fPlyx+P8/0g3bNjguCDQXQiZPn262rRpoyZNmmjo0KGqW7euUlNTlZiYqF9//VX//e9/r3gfVq5cWej3mfj4+Oif//ynYmNj1bhxYz388MOqUaOGjhw5onXr1ikoKEhffPFFgcsvWrRITz/9tOx2u37//XfNnTvXMS89PV25ublatmyZevfurdWrV2v8+PHavXt3oeu8tP1Dhw4t0miLJ9auXet0imf//v0FXsx2tQYNGqSPP/5YI0aM0Lp163TnnXcqNzdXP/30kz7++GOtWLHCaeTnlltuUUxMjNNtxpKc3mAnTZqkdevWKTo6WkOHDtXNN9+s06dPa8eOHVq9erXLBcKrVq3Sr7/+6jjFM2/ePPXq1Ut+fn6SpFdeeUWrVq1SmzZt9Pjjj8vX11czZsxQVlaW47uGpD+O1XHjxqlatWo6cOCADhw44JiXk5OjgwcPatWqVerSpYvuuOMO3XDDDRo8eLBGjRolm82mjz76yOVD4WoFBQWpXbt2eu2113ThwgXVqFFDK1eu9HikZseOHU7Hb76oqCi1bt26wOWCg4P117/+9aoulh02bJjmz5+vESNG6IcffrjstVz169fXkCFDtHXrVlWvXl0ffPCBUlNTNXv2bEedZ555Rp9//rl69Oihhx56SC1btlRmZqa+//57ffLJJ0pOTlbVqlXVr18/vfLKK+rbt6+effZZpaSkaOHChZL+CHkZGRny8fHRlClT1Lp1a8dIXFRUlF555RXFxcUpOTlZvXv3VmBgoA4dOqRPP/1Uw4YN09NPP+1oT34QS05OVv369bVo0SLt2rVLM2fOdFyv4ulrpSg6duyoQYMG6c0339S+ffscp/02bdqkjh07Op0iatGihW655RbHxbq33nprkbeTnp7u9viRdNmg88ILL+ijjz5SUlKS07VeJe4a3zVU6uzevdv069fPhIeHm/Lly5uwsDDTr18/8/333xe6XHHcZpzv7bffNg0bNjTly5c31atXN4899pj57bffXNapQm5VvHS6+DZjY4w5cOCAefDBB01YWJgpX768qVGjhunRo4f55JNPHHWu5HbaS2/VdnfbqTHG7Ny509x3332mSpUqxm63m1q1apm+ffuaNWvWuO3DfPm3PBY25d/y9+STT5p27dqZ5cuXu6ynoNuMAwICzJEjR1y2ebW3GedPAQEB5uabbzZvvPFGget3x5PbjI0xJjs720yePNk0btzY2O12c8MNN5iWLVua+Ph4k56e7qin/3/r5Ny5c029evWM3W43LVq0cHm+jDEmNTXVjBw50kRGRjpeG506dTIzZ8501Ml/vvMnX19fU6tWLTNq1CiXY3jHjh0mJibGVKpUyVSoUMF07NjRfPfdd051inJsX7zv3377rbn99ttNQECAiYiIMM8++6xZsWKF22PwUp7cZvzrr7+ae++911SuXNkEBwebv/zlL+bo0aMuz787l7vN+OJjoaD3jt9++80EBwd7fJvxxZKSkoy/v78ZPXp0ocvXqlXLdO/e3axYscI0bdrU2O1207BhQ7N48WKXumfOnDFxcXHmpptuMn5+fqZq1armjjvuMFOnTnXc2pu/7TvvvNP4+/ubLl26OG4pnjt3rqO8a9eu5tixYy7bWLJkiWnTpo2pWLGiqVixomnYsKEZOXKkSUpKcum3bdu2mdatWxt/f39Tq1Yt8/bbb7usz9PXyqUuvc3YGGNycnLMlClTTMOGDY2fn5+pVq2aiY2NddwifrHXXnvNSDJ/+9vf3D8Bblzu/T9fQe9VxhgzePBgI8lStxkTUFCq1apVy+WN9mLr1q1zebNAwQp607WKywWL2bNnu/3uGxSf/IDiTYsXLy5SiCwqq32/R2GmTZtmbDab+fnnn0u6KSWOa1AAALAAY4xmzZql9u3bq2bNmiXdnBLHNSgo1e69995CrwOqXr06vzhdhgwYMEDVq1cvcH5UVJTTF34BpUFmZqY+//xzrVu3Tt9//70+++yzkm6SJRBQUKq98cYbhc5v1KjRZeug9CjoIsB8bdu2LfD3swCrOnHihPr376/KlSvr+eefL/Cux+uNzZhivqQdAADgKnENCgAAsBwCCgAAsJxSeQ1KXl6ejh49qsDAwCJ9HTEAACh5xhidOXNGERERBf4yfL5SGVCOHj3q8Q+cAQAAa/jll1904403FlqnVAaUwMBASX/s4JX8kjAAALj2MjIyFBkZ6fgcL0ypDCj5p3WCgoIIKAAAlDJFuTyDi2QBAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDlEFAAAIDl+JZ0AwB4V+2xX3lt3cmTuntt3QCub4ygAAAAyyGgAAAAyyGgAAAAyyGgAAAAyyGgAAAAy/E4oGzcuFE9e/ZURESEbDabli1b5jTfZrO5naZMmeKoU7t2bZf5kyZNuuqdAQAAZYPHASUzM1PNmjXT9OnT3c4/duyY0/TBBx/IZrOpT58+TvVeeuklp3pPPvnkle0BAAAoczz+HpTY2FjFxsYWOD8sLMzp8WeffaaOHTuqbt26TuWBgYEudYHrlTe/qwQASiOvXoOSmpqqr776SkOGDHGZN2nSJFWpUkUtWrTQlClTlJOTU+B6srKylJGR4TQBAICyy6vfJPvhhx8qMDBQ9913n1P5qFGjdOuttyokJETfffed4uLidOzYMb3++utu15OQkKD4+HhvNhUAAFiIVwPKBx98oAEDBsjf39+pfMyYMY6/mzZtKj8/Pw0fPlwJCQmy2+0u64mLi3NaJiMjQ5GRkd5rOAAAKFFeCyibNm1SUlKSFi1adNm60dHRysnJUXJysho0aOAy3263uw0uAACgbPLaNSizZs1Sy5Yt1axZs8vW3bVrl3x8fBQaGuqt5gAAgFLE4xGUs2fPav/+/Y7Hhw4d0q5duxQSEqKaNWtK+uMUzOLFi/X3v//dZfnExERt2bJFHTt2VGBgoBITEzV69GgNHDhQN9xww1XsCgAAKCs8Dijbtm1Tx44dHY/zrw0ZPHiw5syZI0lauHChjDHq16+fy/J2u10LFy7UxIkTlZWVpTp16mj06NFO15gAAIDrm80YY0q6EZ7KyMhQcHCw0tPTFRQUVNLNAa5aaf0elORJ3Uu6CQBKEU8+v/ktHgAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDm+Jd0AoDSpPfarkm4CAFwXGEEBAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACW43FA2bhxo3r27KmIiAjZbDYtW7bMaf5DDz0km83mNHXt2tWpzunTpzVgwAAFBQWpcuXKGjJkiM6ePXtVOwIAAMoOjwNKZmammjVrpunTpxdYp2vXrjp27JhjWrBggdP8AQMGaM+ePVq1apW+/PJLbdy4UcOGDfO89QAAoEzy9XSB2NhYxcbGFlrHbrcrLCzM7bwff/xRy5cv19atW3XbbbdJkt566y1169ZNU6dOVUREhKdNAgAAZYxXrkFZv369QkND1aBBAz322GM6deqUY15iYqIqV67sCCeS1LlzZ/n4+GjLli1u15eVlaWMjAynCQAAlF3FHlC6du2qf/3rX1qzZo0mT56sDRs2KDY2Vrm5uZKklJQUhYaGOi3j6+urkJAQpaSkuF1nQkKCgoODHVNkZGRxNxsAAFiIx6d4LueBBx5w/N2kSRM1bdpUUVFRWr9+vTp16nRF64yLi9OYMWMcjzMyMggpAACUYV6/zbhu3bqqWrWq9u/fL0kKCwvT8ePHnerk5OTo9OnTBV63YrfbFRQU5DQBAICyy+sB5ddff9WpU6cUHh4uSWrdurXS0tK0fft2R521a9cqLy9P0dHR3m4OAAAoBTw+xXP27FnHaIgkHTp0SLt27VJISIhCQkIUHx+vPn36KCwsTAcOHNCzzz6rm266STExMZKkRo0aqWvXrho6dKjee+89XbhwQU888YQeeOAB7uABAACSrmAEZdu2bWrRooVatGghSRozZoxatGihF198UeXKldPu3bvVq1cv1a9fX0OGDFHLli21adMm2e12xzrmzZunhg0bqlOnTurWrZvatGmjmTNnFt9eAQCAUs3jEZQOHTrIGFPg/BUrVlx2HSEhIZo/f76nmwYAANcJfosHAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYDgEFAABYjm9JNwBA6VV77FdeWW/ypO5eWS+A0oMRFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkeB5SNGzeqZ8+eioiIkM1m07JlyxzzLly4oOeee05NmjRRxYoVFRERoQcffFBHjx51Wkft2rVls9mcpkmTJl31zgAAgLLB44CSmZmpZs2aafr06S7zzp07px07dmj8+PHasWOHli5dqqSkJPXq1cul7ksvvaRjx445pieffPLK9gAAAJQ5vp4uEBsbq9jYWLfzgoODtWrVKqeyt99+W61atdLhw4dVs2ZNR3lgYKDCwsI83TwAALgOeP0alPT0dNlsNlWuXNmpfNKkSapSpYpatGihKVOmKCcnp8B1ZGVlKSMjw2kCAABll8cjKJ44f/68nnvuOfXr109BQUGO8lGjRunWW29VSEiIvvvuO8XFxenYsWN6/fXX3a4nISFB8fHx3mwqAACwEK8FlAsXLqhv374yxujdd991mjdmzBjH302bNpWfn5+GDx+uhIQE2e12l3XFxcU5LZORkaHIyEhvNR0AAJQwrwSU/HDy888/a+3atU6jJ+5ER0crJydHycnJatCggct8u93uNrgAAICyqdgDSn442bdvn9atW6cqVapcdpldu3bJx8dHoaGhxd0cAABQCnkcUM6ePav9+/c7Hh86dEi7du1SSEiIwsPD9ec//1k7duzQl19+qdzcXKWkpEiSQkJC5Ofnp8TERG3ZskUdO3ZUYGCgEhMTNXr0aA0cOFA33HBD8e0ZAAAotTwOKNu2bVPHjh0dj/OvDRk8eLAmTpyozz//XJLUvHlzp+XWrVunDh06yG63a+HChZo4caKysrJUp04djR492ukaEwAAcH3zOKB06NBBxpgC5xc2T5JuvfVWbd682dPNAgCA6wi/xQMAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACyHgAIAACzH44CyceNG9ezZUxEREbLZbFq2bJnTfGOMXnzxRYWHhysgIECdO3fWvn37nOqcPn1aAwYMUFBQkCpXrqwhQ4bo7NmzV7UjAACg7PA4oGRmZqpZs2aaPn262/mvvfaa3nzzTb333nvasmWLKlasqJiYGJ0/f95RZ8CAAdqzZ49WrVqlL7/8Uhs3btSwYcOufC8AAECZ4uvpArGxsYqNjXU7zxijadOmady4cbrnnnskSf/6179UvXp1LVu2TA888IB+/PFHLV++XFu3btVtt90mSXrrrbfUrVs3TZ06VREREVexOwAAoCwo1mtQDh06pJSUFHXu3NlRFhwcrOjoaCUmJkqSEhMTVblyZUc4kaTOnTvLx8dHW7ZscbverKwsZWRkOE0AAKDsKtaAkpKSIkmqXr26U3n16tUd81JSUhQaGuo039fXVyEhIY46l0pISFBwcLBjioyMLM5mAwAAiykVd/HExcUpPT3dMf3yyy8l3SQAAOBFxRpQwsLCJEmpqalO5ampqY55YWFhOn78uNP8nJwcnT592lHnUna7XUFBQU4TAAAou4o1oNSpU0dhYWFas2aNoywjI0NbtmxR69atJUmtW7dWWlqatm/f7qizdu1a5eXlKTo6ujibAwAASimP7+I5e/as9u/f73h86NAh7dq1SyEhIapZs6aeeuopvfLKK6pXr57q1Kmj8ePHKyIiQr1795YkNWrUSF27dtXQoUP13nvv6cKFC3riiSf0wAMPcAcPAACQdAUBZdu2berYsaPj8ZgxYyRJgwcP1pw5c/Tss88qMzNTw4YNU1pamtq0aaPly5fL39/fscy8efP0xBNPqFOnTvLx8VGfPn305ptvFsPuAACAssBmjDEl3QhPZWRkKDg4WOnp6VyPgmuq9tivSroJ14XkSd1LugkAvMCTz+9ScRcPAAC4vhBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5RBQAACA5fiWdAMA4FK1x37ltXUnT+rutXUDKD6MoAAAAMshoAAAAMshoAAAAMshoAAAAMshoAAAAMshoAAAAMshoAAAAMvhe1BQ5njzOzQAANcGIygAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByij2g1K5dWzabzWUaOXKkJKlDhw4u80aMGFHczQAAAKVYsX9R29atW5Wbm+t4/MMPP6hLly76y1/+4igbOnSoXnrpJcfjChUqFHczAABAKVbsAaVatWpOjydNmqSoqCi1b9/eUVahQgWFhYUV96YBAEAZ4dVrULKzszV37lw98sgjstlsjvJ58+apatWquuWWWxQXF6dz584Vup6srCxlZGQ4TQAAoOzy6m/xLFu2TGlpaXrooYccZf3791etWrUUERGh3bt367nnnlNSUpKWLl1a4HoSEhIUHx/vzaYCAAALsRljjLdWHhMTIz8/P33xxRcF1lm7dq06deqk/fv3Kyoqym2drKwsZWVlOR5nZGQoMjJS6enpCgoKKvZ2o3TjxwJRmORJ3Uu6CcB1KyMjQ8HBwUX6/PbaCMrPP/+s1atXFzoyIknR0dGSVGhAsdvtstvtxd5GAABgTV67BmX27NkKDQ1V9+6F/7eya9cuSVJ4eLi3mgIAAEoZr4yg5OXlafbs2Ro8eLB8ff9vEwcOHND8+fPVrVs3ValSRbt379bo0aPVrl07NW3a1BtNAQAApZBXAsrq1at1+PBhPfLII07lfn5+Wr16taZNm6bMzExFRkaqT58+GjdunDeaAQAASimvBJS7775b7q69jYyM1IYNG7yxSQAAUIbwWzwAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByfEu6AQBwLdUe+5VX1ps8qbtX1gtcrxhBAQAAllPsAWXixImy2WxOU8OGDR3zz58/r5EjR6pKlSqqVKmS+vTpo9TU1OJuBgAAKMW8MoLSuHFjHTt2zDF98803jnmjR4/WF198ocWLF2vDhg06evSo7rvvPm80AwAAlFJeuQbF19dXYWFhLuXp6emaNWuW5s+fr7vuukuSNHv2bDVq1EibN2/W7bff7o3mAACAUsYrIyj79u1TRESE6tatqwEDBujw4cOSpO3bt+vChQvq3Lmzo27Dhg1Vs2ZNJSYmFri+rKwsZWRkOE0AAKDsKvYRlOjoaM2ZM0cNGjTQsWPHFB8fr7Zt2+qHH35QSkqK/Pz8VLlyZadlqlevrpSUlALXmZCQoPj4+OJuKkqYt+6mAACUfsUeUGJjYx1/N23aVNHR0apVq5Y+/vhjBQQEXNE64+LiNGbMGMfjjIwMRUZGXnVbAQCANXn9NuPKlSurfv362r9/v8LCwpSdna20tDSnOqmpqW6vWclnt9sVFBTkNAEAgLLL6wHl7NmzOnDggMLDw9WyZUuVL19ea9asccxPSkrS4cOH1bp1a283BQAAlBLFforn6aefVs+ePVWrVi0dPXpUEyZMULly5dSvXz8FBwdryJAhGjNmjEJCQhQUFKQnn3xSrVu35g4eAADgUOwB5ddff1W/fv106tQpVatWTW3atNHmzZtVrVo1SdIbb7whHx8f9enTR1lZWYqJidE777xT3M0AAAClmM0YY0q6EZ7KyMhQcHCw0tPTuR6lFOMuHpQl/BYPcHmefH7zWzwAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByCCgAAMByij2gJCQk6E9/+pMCAwMVGhqq3r17KykpyalOhw4dZLPZnKYRI0YUd1MAAEApVewBZcOGDRo5cqQ2b96sVatW6cKFC7r77ruVmZnpVG/o0KE6duyYY3rttdeKuykAAKCU8i3uFS5fvtzp8Zw5cxQaGqrt27erXbt2jvIKFSooLCysSOvMyspSVlaW43FGRkbxNBYAAFiS169BSU9PlySFhIQ4lc+bN09Vq1bVLbfcori4OJ07d67AdSQkJCg4ONgxRUZGerXNAACgZNmMMcZbK8/Ly1OvXr2Ulpamb775xlE+c+ZM1apVSxEREdq9e7eee+45tWrVSkuXLnW7HncjKJGRkUpPT1dQUJC3mg8vqz32q5JuAlBskid1L+kmAJaXkZGh4ODgIn1+F/spnouNHDlSP/zwg1M4kaRhw4Y5/m7SpInCw8PVqVMnHThwQFFRUS7rsdvtstvt3mwqAFwVbwZuwg+uR147xfPEE0/oyy+/1Lp163TjjTcWWjc6OlqStH//fm81BwAAlCLFPoJijNGTTz6pTz/9VOvXr1edOnUuu8yuXbskSeHh4cXdHAAAUAoVe0AZOXKk5s+fr88++0yBgYFKSUmRJAUHBysgIEAHDhzQ/Pnz1a1bN1WpUkW7d+/W6NGj1a5dOzVt2rS4mwMAAEqhYg8o7777rqQ/voztYrNnz9ZDDz0kPz8/rV69WtOmTVNmZqYiIyPVp08fjRs3rribAgAASimvnOIpTGRkpDZs2FDcmwUAAGUIv8UDAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsx6u/ZgwAuHre+qVkfiUZVsYICgAAsBwCCgAAsBwCCgAAsByuQUGhvHXuGwCAwjCCAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIeAAgAALIfvQQGA65Q3v+eI3/nB1WIEBQAAWA4BBQAAWA4BBQAAWA4BBQAAWA4BBQAAWA538QAAip237hDi7qDrByMoAADAcggoAADAcggoAADAcrgGpYzw5jdCAgBwrTGCAgAALIeAAgAALIeAAgAALIeAAgAALIeLZAEApYY3bwjgS+CshREUAABgOSU6gjJ9+nRNmTJFKSkpatasmd566y21atWqJJsEALhOlcav5y/LI0olNoKyaNEijRkzRhMmTNCOHTvUrFkzxcTE6Pjx4yXVJAAAYBE2Y4wpiQ1HR0frT3/6k95++21JUl5eniIjI/Xkk09q7NixhS6bkZGh4OBgpaenKygoqNjbxpeeAQCud94YQfHk87tETvFkZ2dr+/btiouLc5T5+Pioc+fOSkxMdKmflZWlrKwsx+P09HRJf+yoN+RlnfPKegEAKC288Rmbv86ijI2USEA5efKkcnNzVb16dafy6tWr66effnKpn5CQoPj4eJfyyMhIr7URAIDrWfA07637zJkzCg4OLrROqbjNOC4uTmPGjHE8zsvL0+nTp1WlShXZbLZi2UZGRoYiIyP1yy+/eOW00fWG/ixe9Gfxoj+LF/1ZvMpyfxpjdObMGUVERFy2bokElKpVq6pcuXJKTU11Kk9NTVVYWJhLfbvdLrvd7lRWuXJlr7QtKCiozB0QJYn+LF70Z/GiP4sX/Vm8ymp/Xm7kJF+J3MXj5+enli1bas2aNY6yvLw8rVmzRq1bty6JJgEAAAspsVM8Y8aM0eDBg3XbbbepVatWmjZtmjIzM/Xwww+XVJMAAIBFlFhAuf/++3XixAm9+OKLSklJUfPmzbV8+XKXC2evFbvdrgkTJricSsKVoT+LF/1ZvOjP4kV/Fi/68w8l9j0oAAAABeG3eAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOVcNwFl+vTpql27tvz9/RUdHa3//Oc/hdZPS0vTyJEjFR4eLrvdrvr16+vrr7++Rq0tHTzt02nTpqlBgwYKCAhQZGSkRo8erfPnz1+j1lrXxo0b1bNnT0VERMhms2nZsmWXXWb9+vW69dZbZbfbddNNN2nOnDleb2dp4Wl/Ll26VF26dFG1atUUFBSk1q1ba8WKFdemsaXElRyj+b799lv5+vqqefPmXmtfaXMl/ZmVlaUXXnhBtWrVkt1uV+3atfXBBx94v7El6LoIKIsWLdKYMWM0YcIE7dixQ82aNVNMTIyOHz/utn52dra6dOmi5ORkffLJJ0pKStL777+vGjVqXOOWW5enfTp//nyNHTtWEyZM0I8//qhZs2Zp0aJFev75569xy60nMzNTzZo10/Tp04tU/9ChQ+revbs6duyoXbt26amnntKjjz7Kh+r/52l/bty4UV26dNHXX3+t7du3q2PHjurZs6d27tzp5ZaWHp72ab60tDQ9+OCD6tSpk5daVjpdSX/27dtXa9as0axZs5SUlKQFCxaoQYMGXmylBZjrQKtWrczIkSMdj3Nzc01ERIRJSEhwW//dd981devWNdnZ2deqiaWOp306cuRIc9dddzmVjRkzxtx5551ebWdpI8l8+umnhdZ59tlnTePGjZ3K7r//fhMTE+PFlpVORelPd26++WYTHx9f/A0qAzzp0/vvv9+MGzfOTJgwwTRr1syr7SqtitKf//73v01wcLA5derUtWmURZT5EZTs7Gxt375dnTt3dpT5+Pioc+fOSkxMdLvM559/rtatW2vkyJGqXr26brnlFv3tb39Tbm7utWq2pV1Jn95xxx3avn274zTQwYMH9fXXX6tbt27XpM1lSWJiolPfS1JMTEyBfQ/P5OXl6cyZMwoJCSnpppRqs2fP1sGDBzVhwoSSbkqp9/nnn+u2227Ta6+9pho1aqh+/fp6+umn9fvvv5d007yqxL7q/lo5efKkcnNzXb5Cv3r16vrpp5/cLnPw4EGtXbtWAwYM0Ndff639+/fr8ccf14ULF3ix6cr6tH///jp58qTatGkjY4xycnI0YsQITvFcgZSUFLd9n5GRod9//10BAQEl1LKyYerUqTp79qz69u1b0k0ptfbt26exY8dq06ZN8vUt8x8zXnfw4EF988038vf316effqqTJ0/q8ccf16lTpzR79uySbp7XlPkRlCuRl5en0NBQzZw5Uy1bttT999+vF154Qe+9915JN63UWr9+vf72t7/pnXfe0Y4dO7R06VJ99dVXevnll0u6aYDD/PnzFR8fr48//lihoaEl3ZxSKTc3V/3791d8fLzq169f0s0pE/Ly8mSz2TRv3jy1atVK3bp10+uvv64PP/ywTI+ilPloW7VqVZUrV06pqalO5ampqQoLC3O7THh4uMqXL69y5co5yho1aqSUlBRlZ2fLz8/Pq222uivp0/Hjx2vQoEF69NFHJUlNmjRRZmamhg0bphdeeEE+PmTlogoLC3Pb90FBQYyeXIWFCxfq0Ucf1eLFi11OoaHozpw5o23btmnnzp164oknJP3xAWuMka+vr1auXKm77rqrhFtZuoSHh6tGjRoKDg52lDVq1EjGGP3666+qV69eCbbOe8r8p4Kfn59atmypNWvWOMry8vK0Zs0atW7d2u0yd955p/bv36+8vDxH2d69exUeHn7dhxPpyvr03LlzLiEkPwAafq/SI61bt3bqe0latWpVgX2Py1uwYIEefvhhLViwQN27dy/p5pRqQUFB+v7777Vr1y7HNGLECDVo0EC7du1SdHR0STex1Lnzzjt19OhRnT171lG2d+9e+fj46MYbbyzBlnlZyV6je20sXLjQ2O12M2fOHPO///3PDBs2zFSuXNmkpKQYY4wZNGiQGTt2rKP+4cOHTWBgoHniiSdMUlKS+fLLL01oaKh55ZVXSmoXLMfTPp0wYYIJDAw0CxYsMAcPHjQrV640UVFRpm/fviW1C5Zx5swZs3PnTrNz504jybz++utm586d5ueffzbGGDN27FgzaNAgR/2DBw+aChUqmGeeecb8+OOPZvr06aZcuXJm+fLlJbULluJpf86bN8/4+vqa6dOnm2PHjjmmtLS0ktoFy/G0Ty/FXTzOPO3PM2fOmBtvvNH8+c9/Nnv27DEbNmww9erVM48++mhJ7cI1cV0EFGOMeeutt0zNmjWNn5+fadWqldm8ebNjXvv27c3gwYOd6n/33XcmOjra2O12U7duXfPqq6+anJyca9xqa/OkTy9cuGAmTpxooqKijL+/v4mMjDSPP/64+e233659wy1m3bp1RpLLlN9/gwcPNu3bt3dZpnnz5sbPz8/UrVvXzJ49+5q326o87c/27dsXWh9XdoxejIDi7Er688cffzSdO3c2AQEB5sYbbzRjxowx586du/aNv4ZsxjC+DgAArKXMX4MCAABKHwIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwnP8HSRVOKMsLUaUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(adequacy_score, bins = 20)\n",
    "plt.title('Отношение длины перевода на EN к референсу EN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e46295ce-fcba-430c-9a13-88d564affdef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T07:00:06.518062Z",
     "iopub.status.busy": "2024-10-18T07:00:06.517791Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from typing import Dict, List\n",
    "import torch\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "717477a4-e4aa-4d29-a1be-1a4ce72256cf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:55:45.863590Z",
     "iopub.status.busy": "2024-10-18T06:55:45.862517Z",
     "iopub.status.idle": "2024-10-18T06:55:45.884302Z",
     "shell.execute_reply": "2024-10-18T06:55:45.883620Z",
     "shell.execute_reply.started": "2024-10-18T06:55:45.863564Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from comet import download_model, load_from_checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "639b918d-fe06-4d91-9992-d4eac39e2f42",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_path_xcomet = download_model(\"Unbabel/XCOMET-XL\")\n",
    "model_xcomet = load_from_checkpoint(model_path_xcomet)\n",
    "model_xcomet = model_xcomet.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0c5d85f-e5ab-4ab3-b895-1717bf0e05e9",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:21:00.197256Z",
     "iopub.status.idle": "2024-10-17T23:21:00.197544Z",
     "shell.execute_reply": "2024-10-17T23:21:00.197433Z",
     "shell.execute_reply.started": "2024-10-17T23:21:00.197403Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def xcomet(data: List[Dict[str, str]]) -> List[float]:\n",
    "    \n",
    "    '''Format\n",
    "    data = [\n",
    "    {\n",
    "        # Source, текст, который надо перевести, src\n",
    "        \"src\": \"В понедельник\", \n",
    "        \n",
    "        # Machine Translation\n",
    "        \"mt\": \"On Monday\", \n",
    "        \n",
    "        # Эталонный перевод, en\n",
    "        \"ref\": \"On Monday\" \n",
    "    }'''\n",
    "    \n",
    "    \n",
    "    xcomet_metric = model_xcomet.predict(data, batch_size=8, gpus=1)\n",
    "    return xcomet_metric.scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f06de50-c0df-4bb5-b76f-d4ab618c7291",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:21:00.198720Z",
     "iopub.status.idle": "2024-10-17T23:21:00.198987Z",
     "shell.execute_reply": "2024-10-17T23:21:00.198882Z",
     "shell.execute_reply.started": "2024-10-17T23:21:00.198866Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "xcomet_cpo_score = xcomet(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89592d35-be01-4ad6-9b1d-16376f16cb59",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:21:00.200727Z",
     "iopub.status.idle": "2024-10-17T23:21:00.201553Z",
     "shell.execute_reply": "2024-10-17T23:21:00.201363Z",
     "shell.execute_reply.started": "2024-10-17T23:21:00.201343Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plt.hist(xcomet_cpo_score, bins = 20)\n",
    "plt.title('XCOMET')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd4f5d69-523c-4055-910b-ef95b76fef4e",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:21:00.202366Z",
     "iopub.status.idle": "2024-10-17T23:21:00.202734Z",
     "shell.execute_reply": "2024-10-17T23:21:00.202573Z",
     "shell.execute_reply.started": "2024-10-17T23:21:00.202553Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# cpo = 0.9626026108213093\n",
    "# modpo = 0.9377332176762843\n",
    "# alma = 0.9512363975580502\n",
    "\n",
    "import numpy as np\n",
    "np.mean(np.array(xcomet_cpo_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dadd35d-789b-449d-adec-9e5e4e7a2f79",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:21:00.203922Z",
     "iopub.status.idle": "2024-10-17T23:21:00.204389Z",
     "shell.execute_reply": "2024-10-17T23:21:00.204248Z",
     "shell.execute_reply.started": "2024-10-17T23:21:00.204228Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "import torch \n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"cointegrated/roberta-large-cola-krishna2020\")\n",
    "model_fluency = AutoModelForSequenceClassification.from_pretrained(\"cointegrated/roberta-large-cola-krishna2020\")\n",
    "model_fluency = model_fluency.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55c69674-fdc9-444d-b314-d470097bdd5f",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:21:00.205205Z",
     "iopub.status.idle": "2024-10-17T23:21:00.205609Z",
     "shell.execute_reply": "2024-10-17T23:21:00.205445Z",
     "shell.execute_reply.started": "2024-10-17T23:21:00.205404Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def fluency(data: str) -> float:\n",
    "    return torch.softmax(model_fluency(**tokenizer(data, return_tensors=\"pt\").to(device)).logits, dim = -1)[0][0].detach().cpu().float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de5365b4-2a13-4c10-801d-e14e3cad792e",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:21:00.206832Z",
     "iopub.status.idle": "2024-10-17T23:21:00.207208Z",
     "shell.execute_reply": "2024-10-17T23:21:00.207050Z",
     "shell.execute_reply.started": "2024-10-17T23:21:00.207030Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_fluency = [item['mt'] for item in data]\n",
    "data_fluency[0], data_fluency[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e97005b-097b-43fa-a567-27aaddc8ddbd",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:21:00.208196Z",
     "iopub.status.idle": "2024-10-17T23:21:00.208552Z",
     "shell.execute_reply": "2024-10-17T23:21:00.208401Z",
     "shell.execute_reply.started": "2024-10-17T23:21:00.208381Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "fluency_cpo_score = [fluency(item) for item in data_fluency]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "755d6dbf-c8b3-4cb5-ab48-fb237c89f7ba",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:21:00.209538Z",
     "iopub.status.idle": "2024-10-17T23:21:00.209863Z",
     "shell.execute_reply": "2024-10-17T23:21:00.209735Z",
     "shell.execute_reply.started": "2024-10-17T23:21:00.209717Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.hist(fluency_cpo_score, bins = 20)\n",
    "plt.title('Fluency')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ac12118-90de-4ee6-bf2d-e179ee7068fa",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:21:00.210576Z",
     "iopub.status.idle": "2024-10-17T23:21:00.210844Z",
     "shell.execute_reply": "2024-10-17T23:21:00.210736Z",
     "shell.execute_reply.started": "2024-10-17T23:21:00.210720Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# cpo = 0.9737626\n",
    "# modpo = 0.93584627\n",
    "# alma = 0.9457284 (??? 0.037422378 ???)\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "np.mean(np.array(fluency_cpo_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95998f92-df40-4aa9-83bd-43c4b7676bf6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-18T06:34:47.430485Z",
     "iopub.status.busy": "2024-10-18T06:34:47.429930Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching 6 files:   0%|          | 0/6 [00:00<?, ?it/s]/home/jupyter/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py:983: UserWarning: Not enough free disk space to download the file. The expected file size is: 42868.10 MB. The target location /tmp/xdg_cache/huggingface/hub/models--Unbabel--wmt23-cometkiwi-da-xxl/blobs only has 35736.35 MB free disk space.\n",
      "  warnings.warn(\n",
      "/home/jupyter/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py:983: UserWarning: Not enough free disk space to download the file. The expected file size is: 42868.10 MB. The target location /tmp/xdg_cache/huggingface/hub/models--Unbabel--wmt23-cometkiwi-da-xxl/blobs only has 35736.35 MB free disk space.\n",
      "  warnings.warn(\n",
      "Fetching 6 files:  50%|█████     | 3/6 [03:33<03:33, 71.02s/it]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/tqdm/contrib/concurrent.py\u001b[0m in \u001b[0;36m_executor_map\u001b[0;34m(PoolExecutor, fn, *iterables, **tqdm_kwargs)\u001b[0m\n\u001b[1;32m     50\u001b[0m                           initargs=(lk,)) as ex:\n\u001b[0;32m---> 51\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0miterables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1181\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1182\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1183\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult_iterator\u001b[0;34m()\u001b[0m\n\u001b[1;32m    620\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 621\u001b[0;31m                         \u001b[0;32myield\u001b[0m \u001b[0m_result_or_cancel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    622\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m_result_or_cancel\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    318\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfut\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    452\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 453\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    454\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    319\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 320\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    321\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_7429/536304295.py\u001b[0m in \u001b[0;36m<cell line: 6>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mmodel_path_kiwi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdownload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Unbabel/wmt23-cometkiwi-da-xxl\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mmodel_kiwi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_from_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_path_kiwi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mmodel_kiwi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_kiwi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/comet/models/__init__.py\u001b[0m in \u001b[0;36mdownload_model\u001b[0;34m(model, saving_directory, local_files_only)\u001b[0m\n\u001b[1;32m     46\u001b[0m ) -> str:\n\u001b[1;32m     47\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m         model_path = snapshot_download(\n\u001b[0m\u001b[1;32m     49\u001b[0m             \u001b[0mrepo_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msaving_directory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_files_only\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlocal_files_only\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         )\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py\u001b[0m in \u001b[0;36m_inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    112\u001b[0m             \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msmoothly_deprecate_use_auth_token\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhas_token\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhas_token\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0m_inner_fn\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/huggingface_hub/_snapshot_download.py\u001b[0m in \u001b[0;36msnapshot_download\u001b[0;34m(repo_id, repo_type, revision, cache_dir, local_dir, library_name, library_version, user_agent, proxies, etag_timeout, force_download, token, local_files_only, allow_patterns, ignore_patterns, max_workers, tqdm_class, headers, endpoint, local_dir_use_symlinks, resume_download)\u001b[0m\n\u001b[1;32m    291\u001b[0m             \u001b[0m_inner_hf_hub_download\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 293\u001b[0;31m         thread_map(\n\u001b[0m\u001b[1;32m    294\u001b[0m             \u001b[0m_inner_hf_hub_download\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m             \u001b[0mfiltered_repo_files\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/tqdm/contrib/concurrent.py\u001b[0m in \u001b[0;36mthread_map\u001b[0;34m(fn, *iterables, **tqdm_kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m     \"\"\"\n\u001b[1;32m     68\u001b[0m     \u001b[0;32mfrom\u001b[0m \u001b[0mconcurrent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfutures\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mThreadPoolExecutor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_executor_map\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mThreadPoolExecutor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0miterables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mtqdm_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/tqdm/contrib/concurrent.py\u001b[0m in \u001b[0;36m_executor_map\u001b[0;34m(PoolExecutor, fn, *iterables, **tqdm_kwargs)\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mensure_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm_class\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlock_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlock_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mlk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0;31m# share lock in case workers are already using `tqdm`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m         with PoolExecutor(max_workers=max_workers, initializer=tqdm_class.set_lock,\n\u001b[0m\u001b[1;32m     50\u001b[0m                           initargs=(lk,)) as ex:\n\u001b[1;32m     51\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0miterables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, exc_type, exc_val, exc_tb)\u001b[0m\n\u001b[1;32m    647\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_tb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 649\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    650\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    651\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/thread.py\u001b[0m in \u001b[0;36mshutdown\u001b[0;34m(self, wait, cancel_futures)\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_threads\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m                 \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m     \u001b[0mshutdown\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_base\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/threading.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1094\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1095\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1096\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_tstate_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1097\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1098\u001b[0m             \u001b[0;31m# the behavior of a negative timeout isn't documented, but\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.10/threading.py\u001b[0m in \u001b[0;36m_wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1115\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1116\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1117\u001b[0m                 \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1118\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# from typing import Dict, List\n",
    "# from comet import download_model, load_from_checkpoint\n",
    "\n",
    "# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "model_path_kiwi = download_model(\"Unbabel/wmt23-cometkiwi-da-xxl\")\n",
    "model_kiwi = load_from_checkpoint(model_path_kiwi)\n",
    "model_kiwi = model_kiwi.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93eca94f-abe6-4c1b-b383-737d387e5ebf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def kiwi(data: List[Dict[str, str]]) -> List[float]:\n",
    "    \n",
    "    '''Format\n",
    "    data = [\n",
    "    {\n",
    "        # source original text, ru\n",
    "        \"src\": \"В понедельник\",\n",
    "        \n",
    "        # machine translation, en\n",
    "        \"mt\": \"On Monday\"\n",
    "    }'''\n",
    "    \n",
    "    \n",
    "    kiwi_metric = model_kiwi.predict(data, batch_size=8, gpus=1)\n",
    "    return kiwi_metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3be6dd38-41bd-4c36-9b91-dc16ff420d8e",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:13:27.564939Z",
     "iopub.status.idle": "2024-10-17T23:13:27.565190Z",
     "shell.execute_reply": "2024-10-17T23:13:27.565089Z",
     "shell.execute_reply.started": "2024-10-17T23:13:27.565074Z"
    }
   },
   "outputs": [],
   "source": [
    "data_kiwi = [(item['src'], item['mt']) for item in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73088d8a-85fb-45dc-bc79-5c5b858482ec",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-10-17T23:13:27.566162Z",
     "iopub.status.idle": "2024-10-17T23:13:27.566413Z",
     "shell.execute_reply": "2024-10-17T23:13:27.566311Z",
     "shell.execute_reply.started": "2024-10-17T23:13:27.566296Z"
    }
   },
   "outputs": [],
   "source": [
    "kiwi_cpo_score = kiwi(data_kiwi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a95e28a-6e5a-4313-bf93-20d595fdf203",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DataSphere Kernel",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}