ALMA_DATASET-2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8efa9481-8e91-496a-adbf-e31bf260018f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T06:45:52.710571Z",
     "iopub.status.busy": "2024-10-10T06:45:52.709233Z",
     "iopub.status.idle": "2024-10-10T06:45:56.394926Z",
     "shell.execute_reply": "2024-10-10T06:45:56.394046Z",
     "shell.execute_reply.started": "2024-10-10T06:45:52.710523Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%pip install datasets -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "969fbb1c-c3d1-4fa4-a6e6-526fceaf9804",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T06:45:56.396980Z",
     "iopub.status.busy": "2024-10-10T06:45:56.396507Z",
     "iopub.status.idle": "2024-10-10T06:45:58.178403Z",
     "shell.execute_reply": "2024-10-10T06:45:58.177610Z",
     "shell.execute_reply.started": "2024-10-10T06:45:56.396944Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eb055f26-0e1b-4685-a06d-9a6176795b93",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T13:19:53.561633Z",
     "iopub.status.busy": "2024-10-09T13:19:53.560651Z",
     "iopub.status.idle": "2024-10-09T13:19:57.006392Z",
     "shell.execute_reply": "2024-10-09T13:19:57.005394Z",
     "shell.execute_reply.started": "2024-10-09T13:19:53.561603Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset = load_dataset(\"haoranxu/ALMA-R-Preference\", \"ru-en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "507eec6b-48b0-483f-8ea9-a962b4a6f009",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T13:02:59.781541Z",
     "iopub.status.busy": "2024-10-09T13:02:59.780568Z",
     "iopub.status.idle": "2024-10-09T13:02:59.876967Z",
     "shell.execute_reply": "2024-10-09T13:02:59.876033Z",
     "shell.execute_reply.started": "2024-10-09T13:02:59.781500Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'dataset' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_2283/48412968.py\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'dataset' is not defined"
     ]
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42e16d5d-ea88-4369-a38d-40483542f9cb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset['train']['translation'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78714a35-6a10-48ba-a68c-b5cfea2bab8a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "len(dataset['train']['translation'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b819cbd-11fd-4d1c-84c9-7ef30129edf7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# for i in range(len(dataset['train']['translation'])):\n",
    "#     if i % 100 == 0:\n",
    "#         print(i)\n",
    "#     current_sample = dataset['train']['translation'][i]\n",
    "#     dataset_new['train']['translation'][i] = {'en': current_sample['en'], # reference ideal translation\n",
    "#                                           'alma_en': current_sample['alma_en'],\n",
    "#                                           'gpt4_en': current_sample['gpt4_en'],\n",
    "#                                           'alma_en_xcomet': current_sample['alma_en_xcomet'],\n",
    "#                                           'gpt4_en_xcomet': current_sample['gpt4_en_xcomet'],\n",
    "#                                           'alma_en_kiwi': current_sample['alma_en_kiwi'],\n",
    "#                                           'gpt4_en_kiwi': current_sample['gpt4_en_kiwi'],\n",
    "#                                           'ru': current_sample['ru']} # text that need to be translated to en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1506d9e6-077e-4123-beb6-98fdda63ad21",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T08:01:30.111940Z",
     "iopub.status.busy": "2024-10-09T08:01:30.110478Z",
     "iopub.status.idle": "2024-10-09T08:04:33.027945Z",
     "shell.execute_reply": "2024-10-09T08:04:33.027115Z",
     "shell.execute_reply.started": "2024-10-09T08:01:30.111893Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>winner</th>\n",
       "      <th>loser</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>В понедельник ученые из Медицинской школы Стэн...</td>\n",
       "      <td>On Monday, researchers at Stanford University'...</td>\n",
       "      <td>On Monday, scientists from Stanford University...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ведущие исследователи утверждают, что он может...</td>\n",
       "      <td>Leading researchers claim that it can help in ...</td>\n",
       "      <td>Leading researchers say it could help detect c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Приблизительно в 9:30 по местному времени (02:...</td>\n",
       "      <td>At around 9:30 local time (02:30 UTC) a JAS 39...</td>\n",
       "      <td>At approximately 9:30 local time (02:30 UTC), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Личность пилота была установлена. Им оказался ...</td>\n",
       "      <td>The pilot's identity has been established. It ...</td>\n",
       "      <td>The identity of the pilot was established. It ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Местные СМИ сообщают, что в аэропорту по пути ...</td>\n",
       "      <td>Local media report that a fire truck flipped o...</td>\n",
       "      <td>Local media reported that a fire truck overtur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004</th>\n",
       "      <td>Так как эти районы являются малонаселенными, и...</td>\n",
       "      <td>Since these areas are sparsely populated, and,...</td>\n",
       "      <td>As these areas are sparsely populated, light p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2005</th>\n",
       "      <td>Японская культура труда иерархичнее и формальн...</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006</th>\n",
       "      <td>Костюмы являются стандартной деловой одеждой, ...</td>\n",
       "      <td>Suits are standard business wear, and colleagu...</td>\n",
       "      <td>Suits are the standard business attire, and co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007</th>\n",
       "      <td>Чрезвычайно важна гармония на рабочем месте, а...</td>\n",
       "      <td>Workplace harmony is extremely important, and ...</td>\n",
       "      <td>Harmony at work is extremely important, and gr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2008</th>\n",
       "      <td>Обычно работники должны получить разрешение на...</td>\n",
       "      <td>Typically, employees must obtain the approval ...</td>\n",
       "      <td>In general, employees are required to get perm...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2009 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    src  ...                                              loser\n",
       "0     В понедельник ученые из Медицинской школы Стэн...  ...  On Monday, scientists from Stanford University...\n",
       "1     Ведущие исследователи утверждают, что он может...  ...  Leading researchers say it could help detect c...\n",
       "2     Приблизительно в 9:30 по местному времени (02:...  ...  At approximately 9:30 local time (02:30 UTC), ...\n",
       "3     Личность пилота была установлена. Им оказался ...  ...  The identity of the pilot was established. It ...\n",
       "4     Местные СМИ сообщают, что в аэропорту по пути ...  ...  Local media reported that a fire truck overtur...\n",
       "...                                                 ...  ...                                                ...\n",
       "2004  Так как эти районы являются малонаселенными, и...  ...  As these areas are sparsely populated, light p...\n",
       "2005  Японская культура труда иерархичнее и формальн...  ...  Japanese work culture is more hierarchical and...\n",
       "2006  Костюмы являются стандартной деловой одеждой, ...  ...  Suits are the standard business attire, and co...\n",
       "2007  Чрезвычайно важна гармония на рабочем месте, а...  ...  Harmony at work is extremely important, and gr...\n",
       "2008  Обычно работники должны получить разрешение на...  ...  In general, employees are required to get perm...\n",
       "\n",
       "[2009 rows x 3 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "xcomet_dataset = pd.DataFrame()\n",
    "src, winner, loser = [], [], []\n",
    "\n",
    "for i in range(len(dataset['train']['translation'])):\n",
    "    if i % 100 == 0:\n",
    "        print(i)\n",
    "    current_sample = dataset['train']['translation'][i]\n",
    "    src += [current_sample['ru']]\n",
    "    if current_sample['alma_en_xcomet'] > current_sample['gpt4_en_xcomet']:\n",
    "        winner += [current_sample['alma_en']]\n",
    "        loser += [current_sample['gpt4_en']]\n",
    "    else:\n",
    "        winner += [current_sample['gpt4_en']]\n",
    "        loser += [current_sample['alma_en']]\n",
    "        \n",
    "xcomet_dataset['src'] = src\n",
    "xcomet_dataset['winner'] = winner\n",
    "xcomet_dataset['loser'] = loser \n",
    "xcomet_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f379c14e-35e1-4195-9565-d20f0d74f865",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T08:05:43.182881Z",
     "iopub.status.busy": "2024-10-09T08:05:43.181759Z",
     "iopub.status.idle": "2024-10-09T08:05:43.233379Z",
     "shell.execute_reply": "2024-10-09T08:05:43.232482Z",
     "shell.execute_reply.started": "2024-10-09T08:05:43.182834Z"
    }
   },
   "outputs": [],
   "source": [
    "xcomet_dataset.to_csv('data/xcomet_dataset.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7b082a24-3a08-45c2-9ea7-55a99e97e3d0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T08:06:56.809571Z",
     "iopub.status.busy": "2024-10-09T08:06:56.808592Z",
     "iopub.status.idle": "2024-10-09T08:09:58.196671Z",
     "shell.execute_reply": "2024-10-09T08:09:58.195705Z",
     "shell.execute_reply.started": "2024-10-09T08:06:56.809530Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>winner</th>\n",
       "      <th>loser</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>В понедельник ученые из Медицинской школы Стэн...</td>\n",
       "      <td>On Monday, scientists from Stanford University...</td>\n",
       "      <td>On Monday, researchers at Stanford University'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ведущие исследователи утверждают, что он может...</td>\n",
       "      <td>Leading researchers claim that it can help in ...</td>\n",
       "      <td>Leading researchers say it could help detect c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Приблизительно в 9:30 по местному времени (02:...</td>\n",
       "      <td>At approximately 9:30 local time (02:30 UTC), ...</td>\n",
       "      <td>At around 9:30 local time (02:30 UTC) a JAS 39...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Личность пилота была установлена. Им оказался ...</td>\n",
       "      <td>The identity of the pilot was established. It ...</td>\n",
       "      <td>The pilot's identity has been established. It ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Местные СМИ сообщают, что в аэропорту по пути ...</td>\n",
       "      <td>Local media report that a fire truck flipped o...</td>\n",
       "      <td>Local media reported that a fire truck overtur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004</th>\n",
       "      <td>Так как эти районы являются малонаселенными, и...</td>\n",
       "      <td>As these areas are sparsely populated, light p...</td>\n",
       "      <td>Since these areas are sparsely populated, and,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2005</th>\n",
       "      <td>Японская культура труда иерархичнее и формальн...</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006</th>\n",
       "      <td>Костюмы являются стандартной деловой одеждой, ...</td>\n",
       "      <td>Suits are standard business wear, and colleagu...</td>\n",
       "      <td>Suits are the standard business attire, and co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007</th>\n",
       "      <td>Чрезвычайно важна гармония на рабочем месте, а...</td>\n",
       "      <td>Workplace harmony is extremely important, and ...</td>\n",
       "      <td>Harmony at work is extremely important, and gr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2008</th>\n",
       "      <td>Обычно работники должны получить разрешение на...</td>\n",
       "      <td>Typically, employees must obtain the approval ...</td>\n",
       "      <td>In general, employees are required to get perm...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2009 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    src  ...                                              loser\n",
       "0     В понедельник ученые из Медицинской школы Стэн...  ...  On Monday, researchers at Stanford University'...\n",
       "1     Ведущие исследователи утверждают, что он может...  ...  Leading researchers say it could help detect c...\n",
       "2     Приблизительно в 9:30 по местному времени (02:...  ...  At around 9:30 local time (02:30 UTC) a JAS 39...\n",
       "3     Личность пилота была установлена. Им оказался ...  ...  The pilot's identity has been established. It ...\n",
       "4     Местные СМИ сообщают, что в аэропорту по пути ...  ...  Local media reported that a fire truck overtur...\n",
       "...                                                 ...  ...                                                ...\n",
       "2004  Так как эти районы являются малонаселенными, и...  ...  Since these areas are sparsely populated, and,...\n",
       "2005  Японская культура труда иерархичнее и формальн...  ...  Japanese work culture is more hierarchical and...\n",
       "2006  Костюмы являются стандартной деловой одеждой, ...  ...  Suits are the standard business attire, and co...\n",
       "2007  Чрезвычайно важна гармония на рабочем месте, а...  ...  Harmony at work is extremely important, and gr...\n",
       "2008  Обычно работники должны получить разрешение на...  ...  In general, employees are required to get perm...\n",
       "\n",
       "[2009 rows x 3 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kiwi_dataset = pd.DataFrame()\n",
    "src, winner, loser = [], [], []\n",
    "\n",
    "for i in range(len(dataset['train']['translation'])):\n",
    "    if i % 100 == 0:\n",
    "        print(i)\n",
    "    current_sample = dataset['train']['translation'][i]\n",
    "    src += [current_sample['ru']]\n",
    "    if current_sample['alma_en_kiwi'] > current_sample['gpt4_en_kiwi']:\n",
    "        winner += [current_sample['alma_en']]\n",
    "        loser += [current_sample['gpt4_en']]\n",
    "    else:\n",
    "        winner += [current_sample['gpt4_en']]\n",
    "        loser += [current_sample['alma_en']]\n",
    "        \n",
    "kiwi_dataset['src'] = src\n",
    "kiwi_dataset['winner'] = winner\n",
    "kiwi_dataset['loser'] = loser \n",
    "kiwi_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4a06b2a7-0095-4a58-be9d-75ca47366618",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T08:11:32.587650Z",
     "iopub.status.busy": "2024-10-09T08:11:32.586590Z",
     "iopub.status.idle": "2024-10-09T08:11:32.647202Z",
     "shell.execute_reply": "2024-10-09T08:11:32.646390Z",
     "shell.execute_reply.started": "2024-10-09T08:11:32.587609Z"
    }
   },
   "outputs": [],
   "source": [
    "kiwi_dataset.to_csv('data/kiwi_dataset.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e012bb92-b612-4598-9c96-3e5e2793bb81",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T13:20:32.513666Z",
     "iopub.status.busy": "2024-10-09T13:20:32.512457Z",
     "iopub.status.idle": "2024-10-09T13:20:36.541062Z",
     "shell.execute_reply": "2024-10-09T13:20:36.540095Z",
     "shell.execute_reply.started": "2024-10-09T13:20:32.513619Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# FLUENCY\n",
    "import torch\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "50f92e52-1b75-4f32-a133-0be589c22674",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T13:20:36.542971Z",
     "iopub.status.busy": "2024-10-09T13:20:36.542326Z",
     "iopub.status.idle": "2024-10-09T13:20:40.226557Z",
     "shell.execute_reply": "2024-10-09T13:20:40.225717Z",
     "shell.execute_reply.started": "2024-10-09T13:20:36.542943Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/.local/lib/python3.10/site-packages/transformers/utils/hub.py:128: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"cointegrated/roberta-large-cola-krishna2020\")\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"cointegrated/roberta-large-cola-krishna2020\")\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7f60289b-4e68-40eb-b57d-10a587b54bcc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:09:34.792065Z",
     "iopub.status.busy": "2024-10-09T11:09:34.791451Z",
     "iopub.status.idle": "2024-10-09T11:09:34.831284Z",
     "shell.execute_reply": "2024-10-09T11:09:34.830485Z",
     "shell.execute_reply.started": "2024-10-09T11:09:34.792025Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[    0,   100, 18774,     7,  3993,   127,  4758,     2]],\n",
       "       device='cuda:0'), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1]], device='cuda:0')}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = 'I forgot to feed my cat'\n",
    "\n",
    "output = tokenizer(text, return_tensors=\"pt\").to(device)\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e96ff609-7bd6-4a71-b887-e86f1dde2eb4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:09:36.763496Z",
     "iopub.status.busy": "2024-10-09T11:09:36.762420Z",
     "iopub.status.idle": "2024-10-09T11:09:38.291176Z",
     "shell.execute_reply": "2024-10-09T11:09:38.290207Z",
     "shell.execute_reply.started": "2024-10-09T11:09:36.763439Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequenceClassifierOutput(loss=None, logits=tensor([[ 2.4507, -2.9105]], device='cuda:0', grad_fn=<AddmmBackward0>), hidden_states=None, attentions=None)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = model(**output)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "208ec9a3-3f0a-4cc9-b6ec-6f148ebb3f87",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:11:09.023122Z",
     "iopub.status.busy": "2024-10-09T11:11:09.022139Z",
     "iopub.status.idle": "2024-10-09T11:11:09.089840Z",
     "shell.execute_reply": "2024-10-09T11:11:09.088949Z",
     "shell.execute_reply.started": "2024-10-09T11:11:09.023080Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.9953, 0.0047]], device='cuda:0', grad_fn=<SoftmaxBackward0>)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.softmax(res.logits, dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4acf31a9-0b20-412d-866a-51d089897d76",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:13:24.183907Z",
     "iopub.status.busy": "2024-10-09T11:13:24.182781Z",
     "iopub.status.idle": "2024-10-09T11:13:24.316241Z",
     "shell.execute_reply": "2024-10-09T11:13:24.315349Z",
     "shell.execute_reply.started": "2024-10-09T11:13:24.183860Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.9953, device='cuda:0', grad_fn=<SelectBackward0>)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.softmax(res.logits, dim=-1)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9e2ca195-674e-4ead-8316-7aec2e9a85f2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:18:37.262985Z",
     "iopub.status.busy": "2024-10-09T11:18:37.261834Z",
     "iopub.status.idle": "2024-10-09T11:23:12.992496Z",
     "shell.execute_reply": "2024-10-09T11:23:12.991664Z",
     "shell.execute_reply.started": "2024-10-09T11:18:37.262941Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "10\n",
      "20\n",
      "30\n",
      "40\n",
      "50\n",
      "60\n",
      "70\n",
      "80\n",
      "90\n",
      "100\n",
      "110\n",
      "120\n",
      "130\n",
      "140\n",
      "150\n",
      "160\n",
      "170\n",
      "180\n",
      "190\n",
      "200\n",
      "210\n",
      "220\n",
      "230\n",
      "240\n",
      "250\n",
      "260\n",
      "270\n",
      "280\n",
      "290\n",
      "300\n",
      "310\n",
      "320\n",
      "330\n",
      "340\n",
      "350\n",
      "360\n",
      "370\n",
      "380\n",
      "390\n",
      "400\n",
      "410\n",
      "420\n",
      "430\n",
      "440\n",
      "450\n",
      "460\n",
      "470\n",
      "480\n",
      "490\n",
      "500\n",
      "510\n",
      "520\n",
      "530\n",
      "540\n",
      "550\n",
      "560\n",
      "570\n",
      "580\n",
      "590\n",
      "600\n",
      "610\n",
      "620\n",
      "630\n",
      "640\n",
      "650\n",
      "660\n",
      "670\n",
      "680\n",
      "690\n",
      "700\n",
      "710\n",
      "720\n",
      "730\n",
      "740\n",
      "750\n",
      "760\n",
      "770\n",
      "780\n",
      "790\n",
      "800\n",
      "810\n",
      "820\n",
      "830\n",
      "840\n",
      "850\n",
      "860\n",
      "870\n",
      "880\n",
      "890\n",
      "900\n",
      "910\n",
      "920\n",
      "930\n",
      "940\n",
      "950\n",
      "960\n",
      "970\n",
      "980\n",
      "990\n",
      "1000\n",
      "1010\n",
      "1020\n",
      "1030\n",
      "1040\n",
      "1050\n",
      "1060\n",
      "1070\n",
      "1080\n",
      "1090\n",
      "1100\n",
      "1110\n",
      "1120\n",
      "1130\n",
      "1140\n",
      "1150\n",
      "1160\n",
      "1170\n",
      "1180\n",
      "1190\n",
      "1200\n",
      "1210\n",
      "1220\n",
      "1230\n",
      "1240\n",
      "1250\n",
      "1260\n",
      "1270\n",
      "1280\n",
      "1290\n",
      "1300\n",
      "1310\n",
      "1320\n",
      "1330\n",
      "1340\n",
      "1350\n",
      "1360\n",
      "1370\n",
      "1380\n",
      "1390\n",
      "1400\n",
      "1410\n",
      "1420\n",
      "1430\n",
      "1440\n",
      "1450\n",
      "1460\n",
      "1470\n",
      "1480\n",
      "1490\n",
      "1500\n",
      "1510\n",
      "1520\n",
      "1530\n",
      "1540\n",
      "1550\n",
      "1560\n",
      "1570\n",
      "1580\n",
      "1590\n",
      "1600\n",
      "1610\n",
      "1620\n",
      "1630\n",
      "1640\n",
      "1650\n",
      "1660\n",
      "1670\n",
      "1680\n",
      "1690\n",
      "1700\n",
      "1710\n",
      "1720\n",
      "1730\n",
      "1740\n",
      "1750\n",
      "1760\n",
      "1770\n",
      "1780\n",
      "1790\n",
      "1800\n",
      "1810\n",
      "1820\n",
      "1830\n",
      "1840\n",
      "1850\n",
      "1860\n",
      "1870\n",
      "1880\n",
      "1890\n",
      "1900\n",
      "1910\n",
      "1920\n",
      "1930\n",
      "1940\n",
      "1950\n",
      "1960\n",
      "1970\n",
      "1980\n",
      "1990\n",
      "2000\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>winner</th>\n",
       "      <th>loser</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>В понедельник ученые из Медицинской школы Стэн...</td>\n",
       "      <td>On Monday, researchers at Stanford University'...</td>\n",
       "      <td>On Monday, scientists from Stanford University...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ведущие исследователи утверждают, что он может...</td>\n",
       "      <td>Leading researchers say it could help detect c...</td>\n",
       "      <td>Leading researchers claim that it can help in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Приблизительно в 9:30 по местному времени (02:...</td>\n",
       "      <td>At around 9:30 local time (02:30 UTC) a JAS 39...</td>\n",
       "      <td>At approximately 9:30 local time (02:30 UTC), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Личность пилота была установлена. Им оказался ...</td>\n",
       "      <td>The identity of the pilot was established. It ...</td>\n",
       "      <td>The pilot's identity has been established. It ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Местные СМИ сообщают, что в аэропорту по пути ...</td>\n",
       "      <td>Local media report that a fire truck flipped o...</td>\n",
       "      <td>Local media reported that a fire truck overtur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004</th>\n",
       "      <td>Так как эти районы являются малонаселенными, и...</td>\n",
       "      <td>Since these areas are sparsely populated, and,...</td>\n",
       "      <td>As these areas are sparsely populated, light p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2005</th>\n",
       "      <td>Японская культура труда иерархичнее и формальн...</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006</th>\n",
       "      <td>Костюмы являются стандартной деловой одеждой, ...</td>\n",
       "      <td>Suits are standard business wear, and colleagu...</td>\n",
       "      <td>Suits are the standard business attire, and co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007</th>\n",
       "      <td>Чрезвычайно важна гармония на рабочем месте, а...</td>\n",
       "      <td>Workplace harmony is extremely important, and ...</td>\n",
       "      <td>Harmony at work is extremely important, and gr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2008</th>\n",
       "      <td>Обычно работники должны получить разрешение на...</td>\n",
       "      <td>Typically, employees must obtain the approval ...</td>\n",
       "      <td>In general, employees are required to get perm...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2009 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    src  ...                                              loser\n",
       "0     В понедельник ученые из Медицинской школы Стэн...  ...  On Monday, scientists from Stanford University...\n",
       "1     Ведущие исследователи утверждают, что он может...  ...  Leading researchers claim that it can help in ...\n",
       "2     Приблизительно в 9:30 по местному времени (02:...  ...  At approximately 9:30 local time (02:30 UTC), ...\n",
       "3     Личность пилота была установлена. Им оказался ...  ...  The pilot's identity has been established. It ...\n",
       "4     Местные СМИ сообщают, что в аэропорту по пути ...  ...  Local media reported that a fire truck overtur...\n",
       "...                                                 ...  ...                                                ...\n",
       "2004  Так как эти районы являются малонаселенными, и...  ...  As these areas are sparsely populated, light p...\n",
       "2005  Японская культура труда иерархичнее и формальн...  ...  Japanese work culture is more hierarchical and...\n",
       "2006  Костюмы являются стандартной деловой одеждой, ...  ...  Suits are the standard business attire, and co...\n",
       "2007  Чрезвычайно важна гармония на рабочем месте, а...  ...  Harmony at work is extremely important, and gr...\n",
       "2008  Обычно работники должны получить разрешение на...  ...  In general, employees are required to get perm...\n",
       "\n",
       "[2009 rows x 3 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "fluency_dataset = pd.DataFrame()\n",
    "src, winner, loser = [], [], []\n",
    "\n",
    "for i in range(len(dataset['train']['translation'])):\n",
    "    if i % 10 == 0:\n",
    "        print(i)\n",
    "    current_sample = dataset['train']['translation'][i]\n",
    "    src += [current_sample['ru']]\n",
    "    \n",
    "    fluency_alma = tokenizer(current_sample['alma_en'], return_tensors=\"pt\").to(device)\n",
    "    fluency_alma = torch.softmax(model(**fluency_alma).logits, dim=-1)[0][0]\n",
    "    \n",
    "    fluency_gpt4 = tokenizer(current_sample['gpt4_en'], return_tensors=\"pt\").to(device)\n",
    "    fluency_gpt4 = torch.softmax(model(**fluency_gpt4).logits, dim=-1)[0][0]\n",
    "    \n",
    "    if fluency_alma > fluency_gpt4:\n",
    "        winner += [current_sample['alma_en']]\n",
    "        loser += [current_sample['gpt4_en']]\n",
    "    else:\n",
    "        winner += [current_sample['gpt4_en']]\n",
    "        loser += [current_sample['alma_en']]\n",
    "        \n",
    "fluency_dataset['src'] = src\n",
    "fluency_dataset['winner'] = winner\n",
    "fluency_dataset['loser'] = loser \n",
    "fluency_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "cdadf2c7-05e1-4d0e-9110-2e8510933432",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:23:40.163007Z",
     "iopub.status.busy": "2024-10-09T11:23:40.161852Z",
     "iopub.status.idle": "2024-10-09T11:23:41.354834Z",
     "shell.execute_reply": "2024-10-09T11:23:41.353896Z",
     "shell.execute_reply.started": "2024-10-09T11:23:40.162959Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "fluency_dataset.to_csv('data/fluency_dataset.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ee00e060-b9c0-420f-8f7b-e0e9e2375c16",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:45:21.265051Z",
     "iopub.status.busy": "2024-10-09T11:45:21.263956Z",
     "iopub.status.idle": "2024-10-09T11:49:59.092203Z",
     "shell.execute_reply": "2024-10-09T11:49:59.091302Z",
     "shell.execute_reply.started": "2024-10-09T11:45:21.265008Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "50\n",
      "100\n",
      "150\n",
      "200\n",
      "250\n",
      "300\n",
      "350\n",
      "400\n",
      "450\n",
      "500\n",
      "550\n",
      "600\n",
      "650\n",
      "700\n",
      "750\n",
      "800\n",
      "850\n",
      "900\n",
      "950\n",
      "1000\n",
      "1050\n",
      "1100\n",
      "1150\n",
      "1200\n",
      "1250\n",
      "1300\n",
      "1350\n",
      "1400\n",
      "1450\n",
      "1500\n",
      "1550\n",
      "1600\n",
      "1650\n",
      "1700\n",
      "1750\n",
      "1800\n",
      "1850\n",
      "1900\n",
      "1950\n",
      "2000\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>winner</th>\n",
       "      <th>loser</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>В понедельник ученые из Медицинской школы Стэн...</td>\n",
       "      <td>On Monday, researchers at Stanford University'...</td>\n",
       "      <td>On Monday, scientists from Stanford University...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ведущие исследователи утверждают, что он может...</td>\n",
       "      <td>Leading researchers claim that it can help in ...</td>\n",
       "      <td>Leading researchers say it could help detect c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Приблизительно в 9:30 по местному времени (02:...</td>\n",
       "      <td>At around 9:30 local time (02:30 UTC) a JAS 39...</td>\n",
       "      <td>At approximately 9:30 local time (02:30 UTC), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Личность пилота была установлена. Им оказался ...</td>\n",
       "      <td>The identity of the pilot was established. It ...</td>\n",
       "      <td>The pilot's identity has been established. It ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Местные СМИ сообщают, что в аэропорту по пути ...</td>\n",
       "      <td>Local media report that a fire truck flipped o...</td>\n",
       "      <td>Local media reported that a fire truck overtur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004</th>\n",
       "      <td>Так как эти районы являются малонаселенными, и...</td>\n",
       "      <td>Since these areas are sparsely populated, and,...</td>\n",
       "      <td>As these areas are sparsely populated, light p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2005</th>\n",
       "      <td>Японская культура труда иерархичнее и формальн...</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006</th>\n",
       "      <td>Костюмы являются стандартной деловой одеждой, ...</td>\n",
       "      <td>Suits are standard business wear, and colleagu...</td>\n",
       "      <td>Suits are the standard business attire, and co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007</th>\n",
       "      <td>Чрезвычайно важна гармония на рабочем месте, а...</td>\n",
       "      <td>Workplace harmony is extremely important, and ...</td>\n",
       "      <td>Harmony at work is extremely important, and gr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2008</th>\n",
       "      <td>Обычно работники должны получить разрешение на...</td>\n",
       "      <td>Typically, employees must obtain the approval ...</td>\n",
       "      <td>In general, employees are required to get perm...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2009 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    src  ...                                              loser\n",
       "0     В понедельник ученые из Медицинской школы Стэн...  ...  On Monday, scientists from Stanford University...\n",
       "1     Ведущие исследователи утверждают, что он может...  ...  Leading researchers say it could help detect c...\n",
       "2     Приблизительно в 9:30 по местному времени (02:...  ...  At approximately 9:30 local time (02:30 UTC), ...\n",
       "3     Личность пилота была установлена. Им оказался ...  ...  The pilot's identity has been established. It ...\n",
       "4     Местные СМИ сообщают, что в аэропорту по пути ...  ...  Local media reported that a fire truck overtur...\n",
       "...                                                 ...  ...                                                ...\n",
       "2004  Так как эти районы являются малонаселенными, и...  ...  As these areas are sparsely populated, light p...\n",
       "2005  Японская культура труда иерархичнее и формальн...  ...  Japanese work culture is more hierarchical and...\n",
       "2006  Костюмы являются стандартной деловой одеждой, ...  ...  Suits are the standard business attire, and co...\n",
       "2007  Чрезвычайно важна гармония на рабочем месте, а...  ...  Harmony at work is extremely important, and gr...\n",
       "2008  Обычно работники должны получить разрешение на...  ...  In general, employees are required to get perm...\n",
       "\n",
       "[2009 rows x 3 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cpo_dataset = pd.DataFrame()\n",
    "src, winner, loser = [], [], []\n",
    "\n",
    "for i in range(len(dataset['train']['translation'])):\n",
    "    if i % 50 == 0:\n",
    "        print(i)\n",
    "        \n",
    "    current_sample = dataset['train']['translation'][i]\n",
    "    src += [current_sample['ru']]\n",
    "    \n",
    "    fluency_alma = tokenizer(current_sample['alma_en'], return_tensors=\"pt\").to(device)\n",
    "    fluency_alma = torch.softmax(model(**fluency_alma).logits, dim=-1)[0][0]\n",
    "    \n",
    "    fluency_gpt4 = tokenizer(current_sample['gpt4_en'], return_tensors=\"pt\").to(device)\n",
    "    fluency_gpt4 = torch.softmax(model(**fluency_gpt4).logits, dim=-1)[0][0]\n",
    "    \n",
    "    fluency_alma = (fluency_alma + current_sample['alma_en_kiwi'] + current_sample['alma_en_xcomet']) / 3\n",
    "    fluency_gpt4 = (fluency_gpt4 + current_sample['gpt4_en_kiwi'] + current_sample['gpt4_en_xcomet']) / 3\n",
    "    \n",
    "    if fluency_alma > fluency_gpt4:\n",
    "        winner += [current_sample['alma_en']]\n",
    "        loser += [current_sample['gpt4_en']]\n",
    "    else:\n",
    "        winner += [current_sample['gpt4_en']]\n",
    "        loser += [current_sample['alma_en']]\n",
    "        \n",
    "        \n",
    "cpo_dataset['src'] = src\n",
    "cpo_dataset['winner'] = winner\n",
    "cpo_dataset['loser'] = loser \n",
    "cpo_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "ba93eab0-2030-4815-9610-fe6e03f43eae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:50:45.958570Z",
     "iopub.status.busy": "2024-10-09T11:50:45.957388Z",
     "iopub.status.idle": "2024-10-09T11:50:46.013360Z",
     "shell.execute_reply": "2024-10-09T11:50:46.012565Z",
     "shell.execute_reply.started": "2024-10-09T11:50:45.958526Z"
    }
   },
   "outputs": [],
   "source": [
    "cpo_dataset.to_csv('data/cpo_dataset.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "e8c0e5dc-0157-46d9-bad2-ae19e40cfd2a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:52:01.047190Z",
     "iopub.status.busy": "2024-10-09T11:52:01.045989Z",
     "iopub.status.idle": "2024-10-09T11:52:05.774220Z",
     "shell.execute_reply": "2024-10-09T11:52:05.773266Z",
     "shell.execute_reply.started": "2024-10-09T11:52:01.047115Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%pip install huggingface_hub -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "1bab865a-28b5-4c90-94e4-2390ef8abdd8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:54:05.545527Z",
     "iopub.status.busy": "2024-10-09T11:54:05.544500Z",
     "iopub.status.idle": "2024-10-09T11:54:05.641640Z",
     "shell.execute_reply": "2024-10-09T11:54:05.640739Z",
     "shell.execute_reply.started": "2024-10-09T11:54:05.545480Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "dataset_cpo = Dataset.from_pandas(cpo_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "99bba189-c2ee-4d79-bbcd-3bd7d8a56386",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T11:54:14.114367Z",
     "iopub.status.busy": "2024-10-09T11:54:14.113370Z",
     "iopub.status.idle": "2024-10-09T11:54:14.129598Z",
     "shell.execute_reply": "2024-10-09T11:54:14.128718Z",
     "shell.execute_reply.started": "2024-10-09T11:54:14.114324Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['src', 'winner', 'loser'],\n",
       "    num_rows: 2009\n",
       "})"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_cpo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b9f58db4-5530-4c22-a3d5-758b713ca55b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T06:52:43.229857Z",
     "iopub.status.busy": "2024-10-10T06:52:43.228538Z",
     "iopub.status.idle": "2024-10-10T06:52:44.650148Z",
     "shell.execute_reply": "2024-10-10T06:52:44.649276Z",
     "shell.execute_reply.started": "2024-10-10T06:52:43.229811Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Delta</th>\n",
       "      <th>alma_en</th>\n",
       "      <th>alma_en_kiwi</th>\n",
       "      <th>alma_en_kiwi_xcomet</th>\n",
       "      <th>alma_en_xcomet</th>\n",
       "      <th>alma_ru</th>\n",
       "      <th>alma_ru_kiwi</th>\n",
       "      <th>alma_ru_kiwi_xcomet</th>\n",
       "      <th>alma_ru_xcomet</th>\n",
       "      <th>en</th>\n",
       "      <th>gpt4_en</th>\n",
       "      <th>gpt4_en_kiwi</th>\n",
       "      <th>gpt4_en_kiwi_xcomet</th>\n",
       "      <th>gpt4_en_xcomet</th>\n",
       "      <th>gpt4_ru</th>\n",
       "      <th>gpt4_ru_kiwi</th>\n",
       "      <th>gpt4_ru_kiwi_xcomet</th>\n",
       "      <th>gpt4_ru_xcomet</th>\n",
       "      <th>language_pair</th>\n",
       "      <th>ref_en_kiwi</th>\n",
       "      <th>ref_en_kiwi_xcomet</th>\n",
       "      <th>ref_en_xcomet</th>\n",
       "      <th>ref_ru_kiwi</th>\n",
       "      <th>ref_ru_kiwi_xcomet</th>\n",
       "      <th>ref_ru_xcomet</th>\n",
       "      <th>required_directions</th>\n",
       "      <th>ru</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>On Monday, researchers at Stanford University'...</td>\n",
       "      <td>0.842576</td>\n",
       "      <td>0.891410</td>\n",
       "      <td>0.940243</td>\n",
       "      <td>В понедельник ученые из Школы медицины Стэнфор...</td>\n",
       "      <td>0.947797</td>\n",
       "      <td>0.947331</td>\n",
       "      <td>0.946866</td>\n",
       "      <td>On Monday, scientists from the Stanford Univer...</td>\n",
       "      <td>On Monday, scientists from Stanford University...</td>\n",
       "      <td>0.863464</td>\n",
       "      <td>0.843561</td>\n",
       "      <td>0.823657</td>\n",
       "      <td>В понедельник учёные из Медицинской школы Стэн...</td>\n",
       "      <td>0.925481</td>\n",
       "      <td>0.934558</td>\n",
       "      <td>0.943635</td>\n",
       "      <td>ru-en</td>\n",
       "      <td>0.831671</td>\n",
       "      <td>0.810386</td>\n",
       "      <td>0.789101</td>\n",
       "      <td>0.941681</td>\n",
       "      <td>0.946758</td>\n",
       "      <td>0.951834</td>\n",
       "      <td></td>\n",
       "      <td>В понедельник ученые из Медицинской школы Стэн...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Leading researchers say it could help detect c...</td>\n",
       "      <td>0.855090</td>\n",
       "      <td>0.843333</td>\n",
       "      <td>0.831575</td>\n",
       "      <td>Основные исследователи говорят, что это может ...</td>\n",
       "      <td>0.924892</td>\n",
       "      <td>0.870960</td>\n",
       "      <td>0.817028</td>\n",
       "      <td>Lead researchers say this may bring early dete...</td>\n",
       "      <td>Leading researchers claim that it can help in ...</td>\n",
       "      <td>0.861727</td>\n",
       "      <td>0.928145</td>\n",
       "      <td>0.994562</td>\n",
       "      <td>Ведущие исследователи говорят, что это может п...</td>\n",
       "      <td>0.976955</td>\n",
       "      <td>0.978656</td>\n",
       "      <td>0.980358</td>\n",
       "      <td>ru-en</td>\n",
       "      <td>0.824746</td>\n",
       "      <td>0.823999</td>\n",
       "      <td>0.823252</td>\n",
       "      <td>0.972896</td>\n",
       "      <td>0.974748</td>\n",
       "      <td>0.976600</td>\n",
       "      <td></td>\n",
       "      <td>Ведущие исследователи утверждают, что он может...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>At approximately 9:30 local time (02:30 UTC), ...</td>\n",
       "      <td>0.879349</td>\n",
       "      <td>0.926376</td>\n",
       "      <td>0.973403</td>\n",
       "      <td>Самолет JAS 39C Gripen рухнул на взлетно-посад...</td>\n",
       "      <td>0.903978</td>\n",
       "      <td>0.935531</td>\n",
       "      <td>0.967085</td>\n",
       "      <td>The JAS 39C Gripen crashed onto a runway at ar...</td>\n",
       "      <td>At around 9:30 local time (02:30 UTC) a JAS 39...</td>\n",
       "      <td>0.866829</td>\n",
       "      <td>0.927372</td>\n",
       "      <td>0.987916</td>\n",
       "      <td>Истребитель JAS 39C Gripen разбился на взлетно...</td>\n",
       "      <td>0.952874</td>\n",
       "      <td>0.972747</td>\n",
       "      <td>0.992621</td>\n",
       "      <td>ru-en</td>\n",
       "      <td>0.859921</td>\n",
       "      <td>0.861010</td>\n",
       "      <td>0.862099</td>\n",
       "      <td>0.896390</td>\n",
       "      <td>0.932251</td>\n",
       "      <td>0.968112</td>\n",
       "      <td></td>\n",
       "      <td>Приблизительно в 9:30 по местному времени (02:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>The pilot's identity has been established. It ...</td>\n",
       "      <td>0.756864</td>\n",
       "      <td>0.830107</td>\n",
       "      <td>0.903351</td>\n",
       "      <td>Пилот был идентифицирован как лейтенант Дилокр...</td>\n",
       "      <td>0.962037</td>\n",
       "      <td>0.953411</td>\n",
       "      <td>0.944785</td>\n",
       "      <td>The pilot was identified as Squadron Leader Di...</td>\n",
       "      <td>The identity of the pilot was established. It ...</td>\n",
       "      <td>0.815404</td>\n",
       "      <td>0.848394</td>\n",
       "      <td>0.881384</td>\n",
       "      <td>Летчика опознали как командира эскадрильи Дило...</td>\n",
       "      <td>0.958193</td>\n",
       "      <td>0.974810</td>\n",
       "      <td>0.991428</td>\n",
       "      <td>ru-en</td>\n",
       "      <td>0.803557</td>\n",
       "      <td>0.782292</td>\n",
       "      <td>0.761027</td>\n",
       "      <td>0.971815</td>\n",
       "      <td>0.983685</td>\n",
       "      <td>0.995555</td>\n",
       "      <td></td>\n",
       "      <td>Личность пилота была установлена. Им оказался ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Local media reported that a fire truck overtur...</td>\n",
       "      <td>0.826231</td>\n",
       "      <td>0.861575</td>\n",
       "      <td>0.896918</td>\n",
       "      <td>Местные СМИ сообщают, что пожарная машина аэро...</td>\n",
       "      <td>0.960159</td>\n",
       "      <td>0.980079</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Local media reports an airport fire vehicle ro...</td>\n",
       "      <td>Local media report that a fire truck flipped o...</td>\n",
       "      <td>0.875460</td>\n",
       "      <td>0.937591</td>\n",
       "      <td>0.999721</td>\n",
       "      <td>Местные СМИ сообщают, что пожарная машина в аэ...</td>\n",
       "      <td>0.955236</td>\n",
       "      <td>0.977618</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>ru-en</td>\n",
       "      <td>0.755580</td>\n",
       "      <td>0.832858</td>\n",
       "      <td>0.910137</td>\n",
       "      <td>0.930592</td>\n",
       "      <td>0.965296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td></td>\n",
       "      <td>Местные СМИ сообщают, что в аэропорту по пути ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004</th>\n",
       "      <td>0.0</td>\n",
       "      <td>As these areas are sparsely populated, light p...</td>\n",
       "      <td>0.890194</td>\n",
       "      <td>0.945097</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Поскольку население здесь малочисленно, и свет...</td>\n",
       "      <td>0.925269</td>\n",
       "      <td>0.962635</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>As the areas are sparsely populated, and light...</td>\n",
       "      <td>Since these areas are sparsely populated, and,...</td>\n",
       "      <td>0.889205</td>\n",
       "      <td>0.944603</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Поскольку районы слабо заселены и, следователь...</td>\n",
       "      <td>0.938765</td>\n",
       "      <td>0.969383</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>ru-en</td>\n",
       "      <td>0.842071</td>\n",
       "      <td>0.904753</td>\n",
       "      <td>0.967435</td>\n",
       "      <td>0.951706</td>\n",
       "      <td>0.975853</td>\n",
       "      <td>1.000000</td>\n",
       "      <td></td>\n",
       "      <td>Так как эти районы являются малонаселенными, и...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2005</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "      <td>0.932158</td>\n",
       "      <td>0.966079</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Японская корпоративная культура более иерархич...</td>\n",
       "      <td>0.967160</td>\n",
       "      <td>0.983580</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "      <td>Japanese work culture is more hierarchical and...</td>\n",
       "      <td>0.933648</td>\n",
       "      <td>0.966824</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Японская рабочая культура более иерархична и ф...</td>\n",
       "      <td>0.452632</td>\n",
       "      <td>0.648631</td>\n",
       "      <td>0.844630</td>\n",
       "      <td>ru-en</td>\n",
       "      <td>0.887933</td>\n",
       "      <td>0.933526</td>\n",
       "      <td>0.979119</td>\n",
       "      <td>0.974342</td>\n",
       "      <td>0.987171</td>\n",
       "      <td>1.000000</td>\n",
       "      <td></td>\n",
       "      <td>Японская культура труда иерархичнее и формальн...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Suits are standard business wear, and colleagu...</td>\n",
       "      <td>0.921347</td>\n",
       "      <td>0.960673</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Костюмы являются стандартной деловой одеждой, ...</td>\n",
       "      <td>0.938116</td>\n",
       "      <td>0.969058</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Suits are standard business attire, and cowork...</td>\n",
       "      <td>Suits are the standard business attire, and co...</td>\n",
       "      <td>0.902686</td>\n",
       "      <td>0.884818</td>\n",
       "      <td>0.866951</td>\n",
       "      <td>Костюмы являются стандартной деловой одеждой, ...</td>\n",
       "      <td>0.907961</td>\n",
       "      <td>0.953980</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>ru-en</td>\n",
       "      <td>0.937788</td>\n",
       "      <td>0.968894</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.935955</td>\n",
       "      <td>0.967977</td>\n",
       "      <td>1.000000</td>\n",
       "      <td></td>\n",
       "      <td>Костюмы являются стандартной деловой одеждой, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Harmony at work is extremely important, and gr...</td>\n",
       "      <td>0.892601</td>\n",
       "      <td>0.946300</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Согласие на рабочем месте является ключевым, а...</td>\n",
       "      <td>0.891888</td>\n",
       "      <td>0.945944</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Workplace harmony is crucial, emphasizing grou...</td>\n",
       "      <td>Workplace harmony is extremely important, and ...</td>\n",
       "      <td>0.920990</td>\n",
       "      <td>0.960495</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Гармония на рабочем месте имеет решающее значе...</td>\n",
       "      <td>0.921245</td>\n",
       "      <td>0.957176</td>\n",
       "      <td>0.993107</td>\n",
       "      <td>ru-en</td>\n",
       "      <td>0.779932</td>\n",
       "      <td>0.886940</td>\n",
       "      <td>0.993947</td>\n",
       "      <td>0.949857</td>\n",
       "      <td>0.974929</td>\n",
       "      <td>1.000000</td>\n",
       "      <td></td>\n",
       "      <td>Чрезвычайно важна гармония на рабочем месте, а...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2008</th>\n",
       "      <td>0.0</td>\n",
       "      <td>In general, employees are required to get perm...</td>\n",
       "      <td>0.924195</td>\n",
       "      <td>0.962097</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Работникам часто приходится получать одобрение...</td>\n",
       "      <td>0.805246</td>\n",
       "      <td>0.858916</td>\n",
       "      <td>0.912586</td>\n",
       "      <td>Workers must often get their superiors' approv...</td>\n",
       "      <td>Typically, employees must obtain the approval ...</td>\n",
       "      <td>0.940581</td>\n",
       "      <td>0.970291</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>Работникам часто необходимо получать одобрение...</td>\n",
       "      <td>0.974603</td>\n",
       "      <td>0.982902</td>\n",
       "      <td>0.991200</td>\n",
       "      <td>ru-en</td>\n",
       "      <td>0.938524</td>\n",
       "      <td>0.904064</td>\n",
       "      <td>0.869604</td>\n",
       "      <td>0.961266</td>\n",
       "      <td>0.980633</td>\n",
       "      <td>1.000000</td>\n",
       "      <td></td>\n",
       "      <td>Обычно работники должны получить разрешение на...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2009 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Delta  ...                                                 ru\n",
       "0       0.0  ...  В понедельник ученые из Медицинской школы Стэн...\n",
       "1       0.0  ...  Ведущие исследователи утверждают, что он может...\n",
       "2       0.0  ...  Приблизительно в 9:30 по местному времени (02:...\n",
       "3       0.0  ...  Личность пилота была установлена. Им оказался ...\n",
       "4       0.0  ...  Местные СМИ сообщают, что в аэропорту по пути ...\n",
       "...     ...  ...                                                ...\n",
       "2004    0.0  ...  Так как эти районы являются малонаселенными, и...\n",
       "2005    0.0  ...  Японская культура труда иерархичнее и формальн...\n",
       "2006    0.0  ...  Костюмы являются стандартной деловой одеждой, ...\n",
       "2007    0.0  ...  Чрезвычайно важна гармония на рабочем месте, а...\n",
       "2008    0.0  ...  Обычно работники должны получить разрешение на...\n",
       "\n",
       "[2009 rows x 27 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pandas import DataFrame\n",
    "\n",
    "dataset = load_dataset(\"marulyanova/ALMA-R-Preference-F\")\n",
    "df = DataFrame(dataset['train']['translation'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bd4a05ad-49f6-441e-a802-01aa7b3b8c0f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T06:48:54.831574Z",
     "iopub.status.busy": "2024-10-10T06:48:54.830533Z",
     "iopub.status.idle": "2024-10-10T06:48:54.933739Z",
     "shell.execute_reply": "2024-10-10T06:48:54.932798Z",
     "shell.execute_reply.started": "2024-10-10T06:48:54.831539Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Delta': 0.0,\n",
       " 'alma_en': \"On Monday, researchers at Stanford University's School of Medicine announced the creation of a new diagnostic tool that can sort cells by type; it's a tiny chip that can be printed out using a standard inkjet printer for about a penny.\",\n",
       " 'alma_en_kiwi': 0.8425758481025696,\n",
       " 'alma_en_kiwi_xcomet': 0.8914096057415009,\n",
       " 'alma_en_xcomet': 0.9402433633804321,\n",
       " 'alma_ru': 'В понедельник ученые из Школы медицины Стэнфордского университета объявили о создании нового диагностического инструмента, который может классифицировать клетки по типу: маленькая печатаемая микросхема, которую можно производить с помощью стандартных струйных принтеров по цене примерно в один цент каждая.',\n",
       " 'alma_ru_kiwi': 0.9477968215942383,\n",
       " 'alma_ru_kiwi_xcomet': 0.9473313689231873,\n",
       " 'alma_ru_xcomet': 0.9468659162521362,\n",
       " 'en': 'On Monday, scientists from the Stanford University School of Medicine announced the invention of a new diagnostic tool that can sort cells by type: a tiny printable chip that can be manufactured using standard inkjet printers for possibly about one U.S. cent each.',\n",
       " 'gpt4_en': \"On Monday, scientists from Stanford University School of Medicine announced the invention of a new diagnostic tool that can sort cells by their type; it's a small chip that can be printed using a standard inkjet printer for about 1 cent USD.\",\n",
       " 'gpt4_en_kiwi': 0.8634641766548157,\n",
       " 'gpt4_en_kiwi_xcomet': 0.843560665845871,\n",
       " 'gpt4_en_xcomet': 0.8236571550369263,\n",
       " 'gpt4_ru': 'В понедельник учёные из Медицинской школы Стэнфордского университета объявили об изобретении нового диагностического инструмента, который может сортировать клетки по типу: это маленькая печатная микросхема, которую можно производить с использованием стандартных струйных принтеров, возможно, за один цент США каждая.',\n",
       " 'gpt4_ru_kiwi': 0.9254813194274902,\n",
       " 'gpt4_ru_kiwi_xcomet': 0.9345579147338867,\n",
       " 'gpt4_ru_xcomet': 0.9436345100402832,\n",
       " 'language_pair': 'ru-en',\n",
       " 'ref_en_kiwi': 0.8316707015037537,\n",
       " 'ref_en_kiwi_xcomet': 0.8103859126567841,\n",
       " 'ref_en_xcomet': 0.7891011238098145,\n",
       " 'ref_ru_kiwi': 0.941681444644928,\n",
       " 'ref_ru_kiwi_xcomet': 0.9467579424381256,\n",
       " 'ref_ru_xcomet': 0.9518344402313232,\n",
       " 'required_directions': '',\n",
       " 'ru': 'В понедельник ученые из Медицинской школы Стэнфордского университета объявили об изобретении нового диагностического инструмента, который может сортировать клетки по их типу; это маленький чип, который можно напечатать, используя стандартный струйный принтер примерно за 1 цент США.'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[\"train\"][\"translation\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "28026d75-82c6-4a2a-8659-9d2b3b6984c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T06:47:04.488048Z",
     "iopub.status.busy": "2024-10-10T06:47:04.486897Z",
     "iopub.status.idle": "2024-10-10T06:47:04.745378Z",
     "shell.execute_reply": "2024-10-10T06:47:04.744196Z",
     "shell.execute_reply.started": "2024-10-10T06:47:04.488004Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'tokenizer' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_2323/628676785.py\u001b[0m in \u001b[0;36m<cell line: 14>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0mcurrent_sample\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'translation'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m     \u001b[0mfluency_alma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcurrent_sample\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'alma_en'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_tensors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"pt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m     \u001b[0mfluency_alma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mfluency_alma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tokenizer' is not defined"
     ]
    }
   ],
   "source": [
    "# DATASET ANALYSIS\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame()\n",
    "gpt4_en_kiwi = []\n",
    "gpt4_en_xcomet = []\n",
    "alma_en_kiwi = []\n",
    "alma_en_xcomet = []\n",
    "gpt4_en_fluency = []\n",
    "alma_en_fluency = []\n",
    "\n",
    "\n",
    "for i in range(len(dataset['train']['translation'])):\n",
    "    if i % 100 == 0:\n",
    "        print(i)\n",
    "        \n",
    "    current_sample = dataset['train']['translation'][i]\n",
    "    \n",
    "    fluency_alma = tokenizer(current_sample['alma_en'], return_tensors=\"pt\").to(device)\n",
    "    fluency_alma = float(torch.softmax(model(**fluency_alma).logits, dim=-1)[0][0].detach().cpu())\n",
    "    \n",
    "    fluency_gpt4 = tokenizer(current_sample['gpt4_en'], return_tensors=\"pt\").to(device)\n",
    "    fluency_gpt4 = float(torch.softmax(model(**fluency_gpt4).logits, dim=-1)[0][0].detach().cpu())\n",
    "    \n",
    "    gpt4_en_kiwi += [current_sample['gpt4_en_kiwi']]\n",
    "    gpt4_en_xcomet += [current_sample['gpt4_en_xcomet']]\n",
    "    alma_en_kiwi += [current_sample['alma_en_kiwi']]\n",
    "    alma_en_xcomet += [current_sample['alma_en_xcomet']]\n",
    "    \n",
    "    gpt4_en_fluency += [fluency_gpt4]\n",
    "    alma_en_fluency += [fluency_alma]\n",
    "\n",
    "df['gpt4_en_kiwi'] = gpt4_en_kiwi\n",
    "df['gpt4_en_xcomet'] = gpt4_en_xcomet\n",
    "\n",
    "df['alma_en_kiwi'] = alma_en_kiwi\n",
    "df['alma_en_xcomet'] = alma_en_xcomet\n",
    "\n",
    "df['gpt4_en_fluency'] = gpt4_en_fluency\n",
    "df['alma_en_fluency'] = alma_en_fluency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "34e109d5-3dc8-4f9e-9dae-926402eedfab",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T07:01:30.898137Z",
     "iopub.status.busy": "2024-10-10T07:01:30.896934Z",
     "iopub.status.idle": "2024-10-10T07:01:31.391081Z",
     "shell.execute_reply": "2024-10-10T07:01:31.390312Z",
     "shell.execute_reply.started": "2024-10-10T07:01:30.898078Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0989317074529326\n",
      "0.09775201254605416\n",
      "not OK\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGzCAYAAACPa3XZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvOUlEQVR4nO3df3RU9Z3/8dckIT8MJIEIM0lJIFU0oOIPkBBFqzSawwk0LKmii11UVmwbUchWJHv4IREMsCIpNPwQMcKaGKULrErFalQ8SAgYpcWqiAqSijPa2mQQmwHJ/f7hl1mHBGHCzGcmyfNxzj2H+dwf855L+fjq537uvTbLsiwBAAAYEhHqAgAAQNdC+AAAAEYRPgAAgFGEDwAAYBThAwAAGEX4AAAARhE+AACAUYQPAABgFOEDAAAYRfiA32w2mx588MFQlwEA6KAIH2hl+fLlstlsysrKCnUpADqI/fv365577tEFF1ygc845R+ecc44GDRqkwsJC/fnPf/Zu9+CDD8pms3mXE9vNnDlTbrdbknzW/9Dy+uuvt6pj27Zt3vV/+9vfTP18+Ckq1AUg/FRWVqp///7auXOnPvroI51//vmhLglAGHvhhRc0fvx4RUVFacKECbr00ksVERGhDz74QBs2bNCKFSu0f/9+9evXz7vPihUr1L17d3399df64x//qPnz5+vVV1/Vm2++qf/+7//2Of66dev08ssvt2ofOHCgz+eWlhZNmTJF8fHxOnLkSPB+MM6eBXzPJ598YkmyNmzYYPXu3dt68MEHW20jyZozZ4754gCEnY8++siKj4+3Bg4caB06dKjV+mPHjlm//e1vrYMHD1qWZVlz5syxJFlffvmlz3bjxo2zJFnbt29vdYzCwkLrTP5ztWLFCis5Odm677772vwOhA8uu8BHZWWlevbsqby8PP385z9XZWXlafc5MYz64Ycf6rbbblNiYqJ69+6tWbNmybIsNTQ0KD8/XwkJCXI4HFq8eLHP/kePHtXs2bM1ZMgQJSYmKj4+Xtdcc41ee+21YP1MAAGyaNEiHTlyRBUVFUpJSWm1PioqSvfee6/S0tJ+8DgjR46U9N3lm/b46quvNHPmTJWUlCgpKaldx4A5hA/4qKys1Lhx4xQdHa1bb71V+/bt065du85o3/Hjx6ulpUULFixQVlaW5s2bp7KyMt1www360Y9+pIULF+r888/Xb37zG73xxhve/dxutx5//HFdd911WrhwoR588EF9+eWXys3N1e7du4P0SwEEwgsvvKDzzz//rOeIffzxx5Kk5OTkdu0/a9YsORwO3X333WdVB8xgzge86uvr9cEHH2jZsmWSpBEjRqhv376qrKzUlVdeedr9hw0bplWrVkmSJk+erP79++s//uM/VFpaqgceeECSdOuttyo1NVVPPPGErr32WklSz549deDAAUVHR3uPdddddykzM1PLli3TmjVrAv1TAQSA2+3WoUOHNHbs2FbrGhsb9e2333o/x8fHKy4uzvv5q6++kiTvnI/ly5fLbrfrmmuu8buOP//5z1q1apX+8Ic/KDIy0v8fAuMY+YBXZWWl7Ha7rr/+eknfzTgfP368qqurdfz48dPu/+///u/eP0dGRmro0KGyLEuTJk3yticlJenCCy/UJ5984rPtieDR0tKir776St9++62GDh2qt99+O1A/D0CAnbg7pXv37q3WXXfdderdu7d3KS8v91l/4YUXqnfv3srIyNDdd9+t888/X5s3b9Y555zjdx333nuvRo0apRtvvLF9PwTGMfIBSdLx48dVXV2t66+/3ueaa1ZWlhYvXqyamprT/sNOT0/3+ZyYmKjY2Fide+65rdr//ve/+7StXbtWixcv1gcffKBjx4552zMyMtr7kwAEWY8ePSR9N3pxslWrVunw4cNyuVy67bbbWq3/n//5HyUkJKhbt27q27evzjvvvHbV8Mwzz2j79u16991327U/QoPwAUnSq6++qs8//1zV1dWqrq5utb6ysvK04aOt4c5TDYFaluX981NPPaXbb79dY8eO1f33368+ffooMjJSpaWl3uvAAMJPYmKiUlJS2vwP/4k5IAcOHGhz32uvvbbV/zFpj/vvv1833XSToqOjvd/V2NgoSWpoaNDRo0eVmpp61t+DwCJ8QNJ34aJPnz6thkYlacOGDdq4caNWrlzpc802UH7/+9/rxz/+sTZs2CCbzeZtnzNnTsC/C0Bg5eXl6fHHH9fOnTs1bNgw49/f0NCgqqoqVVVVtVp3xRVX6NJLL2XiehgifED//Oc/tWHDBt100036+c9/3mp9amqqnn76aT333HMaP358wL//xOiIZVne8FFXV6fa2tpWl3IAhJfp06erqqpKd955p2pqamS3233Wf3+UMxg2btzYqq26ulrPPPOM1q1bp759+wb1+9E+hA/oueee0+HDh/Wzn/2szfXDhw9X7969VVlZGZTwMXr0aG3YsEH/8i//ory8PO3fv18rV67UoEGD2ryWDCB8DBgwQFVVVbr11lt14YUXep9walmW9u/fr6qqKkVERAQtBLR1p82JkY5Ro0YF5NIOAo/wAVVWVio2NlY33HBDm+sjIiKUl5enysrKVhNFA+H222+X0+nUqlWr9NJLL2nQoEF66qmntH79+jbf3QAgvOTn52vPnj1avHix/vjHP+qJJ56QzWZTv379lJeXp1/+8pe69NJLQ10mwojNCvaYGAAAwPfwnA8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGBV2z/loaWnRoUOH1KNHD59HbQMwx7IsHT58WKmpqYqI6Bj/H4W+Awgtf/qNsAsfhw4dUlpaWqjLAKDv3pvRUR5PTd8BhIcz6TfCLnyceEVzQ0ODEhISQlwN0DW53W6lpaV5/z12BPQdQGj502+EXfg4MVyakJBABwKEWEe6fEHfAYSHM+k3OsbFXAAA0GkQPgAAgFGEDwAAYBThAwAAGEX4AAAARhE+AACAUYQPAABgFOEDAAAYRfgAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDz/1n7FZ/WdsDnUZADoY+g7g/xA+AACAUYQPAABgFOEDAAAYRfgAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGEX4AAAARhE+AACAUYQPAABgFOEDAAAYRfgAAABGET4ABN3x48c1a9YsZWRkKC4uTuedd54eeughWZbl3cayLM2ePVspKSmKi4tTTk6O9u3bF8Kqg6v/jM3qP2NzqMsAQoLwASDoFi5cqBUrVuh3v/ud3n//fS1cuFCLFi3SsmXLvNssWrRIS5cu1cqVK1VXV6f4+Hjl5uaqubk5hJUDCIaoUBcAoPPbvn278vPzlZeXJ0nq37+/nn76ae3cuVPSd6MeZWVlmjlzpvLz8yVJ69atk91u16ZNm3TLLbeErHYAgcfIB4Cgu+qqq1RTU6MPP/xQkvSnP/1J27Zt06hRoyRJ+/fvl9PpVE5OjnefxMREZWVlqba2ts1jejweud1unwVAx8DIB4CgmzFjhtxutzIzMxUZGanjx49r/vz5mjBhgiTJ6XRKkux2u89+drvdu+5kpaWlmjt3bnALPwsn5nMcWJAXkO2AzoSRDwBB9+yzz6qyslJVVVV6++23tXbtWj3yyCNau3Ztu49ZXFyspqYm79LQ0BDAigEEEyMfAILu/vvv14wZM7xzNy655BJ9+umnKi0t1cSJE+VwOCRJLpdLKSkp3v1cLpcuu+yyNo8ZExOjmJiYoNcOIPAY+QAQdN98840iIny7m8jISLW0tEiSMjIy5HA4VFNT413vdrtVV1en7Oxso7UCCD5GPgAE3ZgxYzR//nylp6froosu0jvvvKNHH31Ud955pyTJZrNp6tSpmjdvngYMGKCMjAzNmjVLqampGjt2bGiLDxPMDUFnQvgAEHTLli3TrFmz9Otf/1pffPGFUlNTdffdd2v27NnebaZPn64jR45o8uTJamxs1IgRI7RlyxbFxsaGsHIAwUD4ABB0PXr0UFlZmcrKyk65jc1mU0lJiUpKSswVBiAkmPMBAACMYuQDAAKI97UAp8fIBwAAMIrwAQAAjCJ8AAAAo5jzAQBh5OTneTCHBJ0RIx9nqf+MzXQOAAD4gfABAACMInwAAACjCB8AAMAowgcAADCK8AEAAIwifAAAAKMIHwAAwCgeMgYAYYDnBaErYeQDAAAYRfgAAABGET4AAIBRzPkAgCBiLgfQGiMfAADAKMIHAAAwivABAACMYs4HAATAmc7tYA4IwMgHAAAwzK/wcfz4cc2aNUsZGRmKi4vTeeedp4ceekiWZXm3sSxLs2fPVkpKiuLi4pSTk6N9+/YFvHAAANAx+RU+Fi5cqBUrVuh3v/ud3n//fS1cuFCLFi3SsmXLvNssWrRIS5cu1cqVK1VXV6f4+Hjl5uaqubk54MUDAICOx685H9u3b1d+fr7y8vIkSf3799fTTz+tnTt3Svpu1KOsrEwzZ85Ufn6+JGndunWy2+3atGmTbrnllgCXDwAAOhq/Rj6uuuoq1dTU6MMPP5Qk/elPf9K2bds0atQoSdL+/fvldDqVk5Pj3ScxMVFZWVmqra1t85gej0dut9tnAQAAnZdfIx8zZsyQ2+1WZmamIiMjdfz4cc2fP18TJkyQJDmdTkmS3W732c9ut3vXnay0tFRz585tT+1h5cQM9gML8kJcCQAA4c2vkY9nn31WlZWVqqqq0ttvv621a9fqkUce0dq1a9tdQHFxsZqamrxLQ0NDu48FAADCn18jH/fff79mzJjhnbtxySWX6NNPP1VpaakmTpwoh8MhSXK5XEpJSfHu53K5dNlll7V5zJiYGMXExLSzfAAA0NH4NfLxzTffKCLCd5fIyEi1tLRIkjIyMuRwOFRTU+Nd73a7VVdXp+zs7ACUC6Aj6t+/v2w2W6ulsLBQktTc3KzCwkIlJyere/fuKigokMvlCnHVAILFr5GPMWPGaP78+UpPT9dFF12kd955R48++qjuvPNOSZLNZtPUqVM1b948DRgwQBkZGZo1a5ZSU1M1duzYYNQfNMzhAAJn165dOn78uPfzu+++qxtuuEE33XSTJGnatGnavHmz1q9fr8TERN1zzz0aN26c3nzzzVCVDCCI/Aofy5Yt06xZs/TrX/9aX3zxhVJTU3X33Xdr9uzZ3m2mT5+uI0eOaPLkyWpsbNSIESO0ZcsWxcbGBrx4AB1D7969fT4vWLBA5513nn7yk5+oqalJa9asUVVVlUaOHClJqqio0MCBA7Vjxw4NHz68zWN6PB55PB7vZ+6UAzoOvy679OjRQ2VlZfr000/1z3/+Ux9//LHmzZun6Oho7zY2m00lJSVyOp1qbm7WK6+8ogsuuCDghQPomI4ePaqnnnpKd955p2w2m+rr63Xs2DGfW/QzMzOVnp5+ylv0pe/ulEtMTPQuaWlpJsoHEAC82wWAUZs2bVJjY6Nuv/12Sd/doh8dHa2kpCSf7X7oFn2JO+WAjoy32gIwas2aNRo1apRSU1PP6jjcKQd0XIQPAMZ8+umneuWVV7RhwwZvm8Ph0NGjR9XY2Ogz+uFyuby37wPoXLjsAsCYiooK9enTx/t+KEkaMmSIunXr5nOL/t69e3Xw4EFu0Qc6KUY+ToNbboHAaGlpUUVFhSZOnKioqP/rehITEzVp0iQVFRWpV69eSkhI0JQpU5SdnX3KO13CAX0D0H6EDwBGvPLKKzp48KD3uUDft2TJEkVERKigoEAej0e5ublavnx5CKoEYALhA4ARN954oyzLanNdbGysysvLVV5ebrgqAKHAnA8AAGAU4QMAOrD+MzZ7558AHQXhAwAAGEX4AAAARhE+DGJ4FAAAwgcAADCM8AEAAIwifAAAAKN4yFiAfX9OB49dBgCgNcIHAIQhJqejM+OyCwAAMIrwAQAAjOKySzsxJAoAQPsw8gEAAIwifAAAAKMIHwAAwCjCBwAAMIrwAQAAjCJ8AAAAowgfAADAKMIHAAAwivABAACMInwAAACjuuzj1U88Hv3k194H47HppzrmqWoAAKAzY+QDAAAYRfgAAABGddnLLgDQkfFmbXRkhI8zxD90AAACg8suAADAKMIHAAAwivABwIjPPvtMt912m5KTkxUXF6dLLrlEb731lne9ZVmaPXu2UlJSFBcXp5ycHO3bty+EFQMIFsIHgKD7xz/+oauvvlrdunXTiy++qPfee0+LFy9Wz549vdssWrRIS5cu1cqVK1VXV6f4+Hjl5uaqubk5hJUDCAYmnAIIuoULFyotLU0VFRXetoyMDO+fLctSWVmZZs6cqfz8fEnSunXrZLfbtWnTJt1yyy3GawYQPIx8AAi65557TkOHDtVNN92kPn366PLLL9fq1au96/fv3y+n06mcnBxvW2JiorKyslRbW9vmMT0ej9xut88CoGNg5COIuD0X+M4nn3yiFStWqKioSP/5n/+pXbt26d5771V0dLQmTpwop9MpSbLb7T772e1277qTlZaWau7cuUGv/fvC+d80r2tAR8LIB4Cga2lp0RVXXKGHH35Yl19+uSZPnqy77rpLK1eubPcxi4uL1dTU5F0aGhoCWDGAYCJ8AAi6lJQUDRo0yKdt4MCBOnjwoCTJ4XBIklwul882LpfLu+5kMTExSkhI8FkAdAyEDwBBd/XVV2vv3r0+bR9++KH69esn6bvJpw6HQzU1Nd71brdbdXV1ys7ONlorgOBjzgeAoJs2bZquuuoqPfzww7r55pu1c+dOPfbYY3rsscckSTabTVOnTtW8efM0YMAAZWRkaNasWUpNTdXYsWNDW/xphPM8ECBcET4ABN2VV16pjRs3qri4WCUlJcrIyFBZWZkmTJjg3Wb69Ok6cuSIJk+erMbGRo0YMUJbtmxRbGxsCCsHEAyEDwBGjB49WqNHjz7lepvNppKSEpWUlBisCkAoMOcjDPSfsZmhWwBAl0H4AAAARhE+AACAUYQPAABgFBNO/z/mXAAAYAYjHwAAwCjCBwAAMIrwAQAAjCJ8AAAAowgfAADAKMIHAAAwivABAACMInyEEd7xAgDoCggfAADAKMIHAAAwyu/w8dlnn+m2225TcnKy4uLidMkll+itt97yrrcsS7Nnz1ZKSori4uKUk5Ojffv2BbRoAADQcfkVPv7xj3/o6quvVrdu3fTiiy/qvffe0+LFi9WzZ0/vNosWLdLSpUu1cuVK1dXVKT4+Xrm5uWpubg548QAAoOPx68VyCxcuVFpamioqKrxtGRkZ3j9blqWysjLNnDlT+fn5kqR169bJbrdr06ZNuuWWWwJUNgAA6Kj8Gvl47rnnNHToUN10003q06ePLr/8cq1evdq7fv/+/XI6ncrJyfG2JSYmKisrS7W1tW0e0+PxyO12+ywAgPY5cdccd84hnPkVPj755BOtWLFCAwYM0EsvvaRf/epXuvfee7V27VpJktPplCTZ7Xaf/ex2u3fdyUpLS5WYmOhd0tLS2vM7AABAB+FX+GhpadEVV1yhhx9+WJdffrkmT56su+66SytXrmx3AcXFxWpqavIuDQ0N7T4WAAAIf36Fj5SUFA0aNMinbeDAgTp48KAkyeFwSJJcLpfPNi6Xy7vuZDExMUpISPBZAABA5+VX+Lj66qu1d+9en7YPP/xQ/fr1k/Td5FOHw6Gamhrverfbrbq6OmVnZwegXAAA0NH5dbfLtGnTdNVVV+nhhx/WzTffrJ07d+qxxx7TY489Jkmy2WyaOnWq5s2bpwEDBigjI0OzZs1Samqqxo4dG4z6AQBAB+NX+Ljyyiu1ceNGFRcXq6SkRBkZGSorK9OECRO820yfPl1HjhzR5MmT1djYqBEjRmjLli2KjY0NePEAAKDj8St8SNLo0aM1evToU6632WwqKSlRSUnJWRUGAAA6J97tAgAAjCJ8AAAAowgfAADAKMIHAAAwivABIOgefPBB2Ww2nyUzM9O7vrm5WYWFhUpOTlb37t1VUFDQ6mGFADoPwgcAIy666CJ9/vnn3mXbtm3eddOmTdPzzz+v9evXa+vWrTp06JDGjRsXwmoBBJPft9oCQHtERUW1+ZqFpqYmrVmzRlVVVRo5cqQkqaKiQgMHDtSOHTs0fPhw06UCCDJGPgAYsW/fPqWmpurHP/6xJkyY4H0nVH19vY4dO6acnBzvtpmZmUpPT1dtbe0pj+fxeOR2u30WAB0DIx8h0H/G5lCXABiVlZWlJ598UhdeeKE+//xzzZ07V9dcc43effddOZ1ORUdHKykpyWcfu90up9N5ymOWlpZq7ty5Qa68YzvR1xxYkBfiSgBfhA8AQTdq1CjvnwcPHqysrCz169dPzz77rOLi4tp1zOLiYhUVFXk/u91upaWlnXWtAIKPyy4AjEtKStIFF1ygjz76SA6HQ0ePHlVjY6PPNi6Xq805IifExMQoISHBZwHQMRA+ABj39ddf6+OPP1ZKSoqGDBmibt26qaamxrt+7969OnjwoLKzs0NYJYBg4bILgKD7zW9+ozFjxqhfv346dOiQ5syZo8jISN16661KTEzUpEmTVFRUpF69eikhIUFTpkxRdnY2d7oAnRThA0DQ/fWvf9Wtt96qv//97+rdu7dGjBihHTt2qHfv3pKkJUuWKCIiQgUFBfJ4PMrNzdXy5ctDXDWAYCF8AAi66urqH1wfGxur8vJylZeXG6oIQCgx5wMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGEX4AAAARhE+AOAU+s/YzIsggSAgfAAAAKMIHwAAwCjCBwAAMIrwAQAAjOpy73Zh8hgAAKHFyAcAADCK8AEAAIwifAAAAKMIHx0ADzoCAHQmhA8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGNVlHq9+qltVuYUVAACzGPkAAABGET4AAIBRhA8AAGBUl5nz0ZGcmIdyYEFeiCsBEG6Yp4bOgJEPAABgFOEDAAAYRfgAAABGET4AAIBRhA8ARi1YsEA2m01Tp071tjU3N6uwsFDJycnq3r27CgoK5HK5QlckgKAifAAwZteuXVq1apUGDx7s0z5t2jQ9//zzWr9+vbZu3apDhw5p3LhxIaoSQLARPgAY8fXXX2vChAlavXq1evbs6W1vamrSmjVr9Oijj2rkyJEaMmSIKioqtH37du3YsSOEFQMIFsIHACMKCwuVl5ennJwcn/b6+nodO3bMpz0zM1Pp6emqra095fE8Ho/cbrfPAqBj4CFjAIKuurpab7/9tnbt2tVqndPpVHR0tJKSknza7Xa7nE7nKY9ZWlqquXPnBrrUNvFgLyCwGPkAEFQNDQ267777VFlZqdjY2IAdt7i4WE1NTd6loaEhYMcGEFyEDwBBVV9fry+++EJXXHGFoqKiFBUVpa1bt2rp0qWKioqS3W7X0aNH1djY6LOfy+WSw+E45XFjYmKUkJDgswDoGLjsAiCofvrTn2rPnj0+bXfccYcyMzP1wAMPKC0tTd26dVNNTY0KCgokSXv37tXBgweVnZ0dipIBBBnhA0BQ9ejRQxdffLFPW3x8vJKTk73tkyZNUlFRkXr16qWEhARNmTJF2dnZGj58eChKBhBkhA8AIbdkyRJFRESooKBAHo9Hubm5Wr58eajLAhAkhA8Axr3++us+n2NjY1VeXq7y8vLQFATAKCacAgAAowgfAADAKMIHAAAwijkfYYynKgIAOiNGPgAAgFGEDwAAYNRZhY8FCxbIZrNp6tSp3rbm5mYVFhYqOTlZ3bt3V0FBgVwu19nWecb6z9jM5QoAAMJYu8PHrl27tGrVKg0ePNinfdq0aXr++ee1fv16bd26VYcOHdK4cePOulAAANA5tCt8fP3115owYYJWr16tnj17etubmpq0Zs0aPfrooxo5cqSGDBmiiooKbd++XTt27AhY0QAAoONqV/goLCxUXl6ecnJyfNrr6+t17Ngxn/bMzEylp6ertra2zWN5PB653W6fBQAQeFyWRrjw+1bb6upqvf3229q1a1erdU6nU9HR0UpKSvJpt9vtcjqdbR6vtLRUc+fO9bcMAADQQfk18tHQ0KD77rtPlZWVio2NDUgBxcXFampq8i4NDQ0BOS4AAAhPfoWP+vp6ffHFF7riiisUFRWlqKgobd26VUuXLlVUVJTsdruOHj2qxsZGn/1cLpccDkebx4yJiVFCQoLPAgAAOi+/Lrv89Kc/1Z49e3za7rjjDmVmZuqBBx5QWlqaunXrppqaGhUUFEiS9u7dq4MHDyo7OztwVQMAgA7Lr/DRo0cPXXzxxT5t8fHxSk5O9rZPmjRJRUVF6tWrlxISEjRlyhRlZ2dr+PDhgasaAAB0WAF/t8uSJUsUERGhgoICeTwe5ebmavny5YH+GgAA0EGddfh4/fXXfT7HxsaqvLxc5eXlZ3toAADQCfFuFwAAYBThAwAAGEX4AAAARhE+AACAUQG/2yVUeF8BAAAdAyMfAADAKMIHAAAwivABAACMInx0IP1nbGZuCwCgw+s0E04BIFA6e8g/8fsOLMgLcSXoqhj5AAAARhE+AACAUR3+sktnHx4FAKCzYeQDQNCtWLFCgwcPVkJCghISEpSdna0XX3zRu765uVmFhYVKTk5W9+7dVVBQIJfLFbR6mLwNhBbhA0DQ9e3bVwsWLFB9fb3eeustjRw5Uvn5+frLX/4iSZo2bZqef/55rV+/Xlu3btWhQ4c0bty4EFcNIFg6/GUXAOFvzJgxPp/nz5+vFStWaMeOHerbt6/WrFmjqqoqjRw5UpJUUVGhgQMHaseOHRo+fHgoSgYQRIx8ADDq+PHjqq6u1pEjR5Sdna36+nodO3ZMOTk53m0yMzOVnp6u2traUx7H4/HI7Xb7LAA6BsIHACP27Nmj7t27KyYmRr/85S+1ceNGDRo0SE6nU9HR0UpKSvLZ3m63y+l0nvJ4paWlSkxM9C5paWl+18TcDyA0CB8AjLjwwgu1e/du1dXV6Ve/+pUmTpyo9957r93HKy4uVlNTk3dpaGgIYLUAgqnTzvngCX5AeImOjtb5558vSRoyZIh27dql3/72txo/fryOHj2qxsZGn9EPl8slh8NxyuPFxMQoJiYm2GUDCAJGPgCEREtLizwej4YMGaJu3bqppqbGu27v3r06ePCgsrOzQ1ghgGDptCMfAMJHcXGxRo0apfT0dB0+fFhVVVV6/fXX9dJLLykxMVGTJk1SUVGRevXqpYSEBE2ZMkXZ2dnc6RIgp5vXwkgxTCN8AAi6L774Qv/2b/+mzz//XImJiRo8eLBeeukl3XDDDZKkJUuWKCIiQgUFBfJ4PMrNzdXy5ctDXDWAYCF8AAi6NWvW/OD62NhYlZeXq7y83FBFAEKJOR8AAMAowgcAADCq01924QFCAACEF0Y+AACAUYQPAABgFOEDAAAYRfgAAABGET4AAIBRhA8AAGAU4QMAABhF+ADQ5fWfsZlnAgEGET4AAIBRhA8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGEX4AAAARhE+AACAUYQPAABgFOGjA+JR0ACAjiwq1AUAQLgg1ANmMPIBAACMInwAAACjCB8AAMAowgcAADCK8AEAAIwifAAAAKMIHwAAwCjCBwAAMIrwASDoSktLdeWVV6pHjx7q06ePxo4dq7179/ps09zcrMLCQiUnJ6t79+4qKCiQy+UKUcUAgonw0QnwuHWEu61bt6qwsFA7duzQyy+/rGPHjunGG2/UkSNHvNtMmzZNzz//vNavX6+tW7fq0KFDGjduXAirBhAsPF4dQNBt2bLF5/OTTz6pPn36qL6+Xtdee62ampq0Zs0aVVVVaeTIkZKkiooKDRw4UDt27NDw4cNbHdPj8cjj8Xg/u93u4P4IAAHDyAcA45qamiRJvXr1kiTV19fr2LFjysnJ8W6TmZmp9PR01dbWtnmM0tJSJSYmepe0tLTgF95FMJqKYCN8ADCqpaVFU6dO1dVXX62LL75YkuR0OhUdHa2kpCSfbe12u5xOZ5vHKS4uVlNTk3dpaGgIdukAAoTLLgCMKiws1Lvvvqtt27ad1XFiYmIUExMToKoAmMTIBwBj7rnnHr3wwgt67bXX1LdvX2+7w+HQ0aNH1djY6LO9y+WSw+EwXCWAYCN8AAg6y7J0zz33aOPGjXr11VeVkZHhs37IkCHq1q2bampqvG179+7VwYMHlZ2dbbpcAEHmV/jgXn0A7VFYWKinnnpKVVVV6tGjh5xOp5xOp/75z39KkhITEzVp0iQVFRXptddeU319ve644w5lZ2e3eacLgI7Nr/DBvfoA2mPFihVqamrSddddp5SUFO/yzDPPeLdZsmSJRo8erYKCAl177bVyOBzasGFDCKsGECx+TTgNxr36ADo/y7JOu01sbKzKy8tVXl5uoCIAoXRWcz4Cca++x+OR2+32WQAAQOfV7lttA3WvfmlpqebOndveMnAGTjws6MCCvBBXAiCc8CAxhEq7Rz5O3KtfXV19VgXwoCAAALqWdo18nLhX/4033jjlvfrfH/34oXv1eVAQAABdi18jH9yrDwAAzpZfIx+FhYWqqqrS//7v/3rv1Ze+u0c/Li7O5179Xr16KSEhQVOmTOFefQAA4OVX+FixYoUk6brrrvNpr6io0O233y7pu3v1IyIiVFBQII/Ho9zcXC1fvjwgxQIAgI7Pr/DBvfoAAOBs8W4XAABgFOEDAAAYRfgAAABGET4AAIBRhA8AAGAU4QMA8IP6z9jMe2AQUIQPAABgFOEDAAAY1a4Xy6FjYJgUABCOGPkAAABGET4AAIBRhA8AAGAUcz46sJPndJz4fGBBXijKAQDgjBA+AACSmKQOc7jsAgAAjCJ8AAAAowgfAADAKMIHAAAwivABAACMInwAAACjCB8AAMAowgcAADCK8AEg6N544w2NGTNGqampstls2rRpk896y7I0e/ZspaSkKC4uTjk5Odq3b19oigUQdIQPAEF35MgRXXrppSovL29z/aJFi7R06VKtXLlSdXV1io+PV25urpqbmw1XCsAEHq8OIOhGjRqlUaNGtbnOsiyVlZVp5syZys/PlyStW7dOdrtdmzZt0i233GKyVAAGMPIBIKT2798vp9OpnJwcb1tiYqKysrJUW1t7yv08Ho/cbrfPAqBjIHwACCmn0ylJstvtPu12u927ri2lpaVKTEz0LmlpaUGtE9+9eI6XzyEQCB+dEB0EuoLi4mI1NTV5l4aGhlCXBOAMET4AhJTD4ZAkuVwun3aXy+Vd15aYmBglJCT4LAA6BsIHgJDKyMiQw+FQTU2Nt83tdquurk7Z2dkhrAxAsHC3Sxdy4lLMgQV5Ia4EXc3XX3+tjz76yPt5//792r17t3r16qX09HRNnTpV8+bN04ABA5SRkaFZs2YpNTVVY8eODV3R4PItgobwASDo3nrrLV1//fXez0VFRZKkiRMn6sknn9T06dN15MgRTZ48WY2NjRoxYoS2bNmi2NjYUJUMIIgIHwCC7rrrrpNlWadcb7PZVFJSopKSEoNVAQgV5nwAAACjGPnogr5/HZf5HwAA0xj5AAAARhE+AACAUYQPAABgFOEDAAAYRfgAAABGET4AAIBR3GqLM3LyY5a5RRcA0F6MfAAA2qX/jM28/wXtQvgAAABGET4AAIBRhI8ujmFTAIBphA8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAQFNxNh1Ph8erwcaKjONPHp/PYdQCAvxj5AAAARhE+AACAUVx2AQD45eTLrWd6udbfy7rovAgfkNS6Mzld+5ke7+ROhs4HAMBlFwAAYBThAwAAGMVlFwBAQJzp5dvvfz7VJdhT3cZ/tpduufQbHggfaJdwenAQnQkAdCxcdgEAAEYRPgAAgFFBu+xSXl6u//qv/5LT6dSll16qZcuWadiwYcH6OgCdAP1G19XeVzucPBfkZKdbf7r9AlHr2QrlpeVgfXdQwsczzzyjoqIirVy5UllZWSorK1Nubq727t2rPn36BOMrEWCBntNxun/4PzSZ7HTPDDndd5qY0HamHVhn7EQChX4D6DqCctnl0Ucf1V133aU77rhDgwYN0sqVK3XOOefoiSeeCMbXAegE6DeAriPgIx9Hjx5VfX29iouLvW0RERHKyclRbW1tq+09Ho88Ho/3c1NTkyTJ7Xaf0fe1eL45y4oRSCf+3k78vZz8+Uy3+/7ff1ttP3TM0+13uv1Pt9+ZHKu9NQWTP999YhvLsoJa0wn+9hvS2fUd9Bvho719xunaA7VfW0z/O+6U/YYVYJ999pklydq+fbtP+/33328NGzas1fZz5syxJLGwsITh0tDQEOguIiD9hmXRd7CwhOtyJv1GyJ/zUVxcrKKiIu/nlpYWffXVV0pOTpbNZvvBfd1ut9LS0tTQ0KCEhIRgl9qhca7809XPl2VZOnz4sFJTU0Ndyim1t+/o6n+3/uJ8+a+rnjN/+o2Ah49zzz1XkZGRcrlcPu0ul0sOh6PV9jExMYqJifFpS0pK8us7ExISutRf8NngXPmnK5+vxMREY9/lb78hnX3f0ZX/btuD8+W/rnjOzrTfCPiE0+joaA0ZMkQ1NTXetpaWFtXU1Cg7OzvQXwegE6DfALqWoFx2KSoq0sSJEzV06FANGzZMZWVlOnLkiO64445gfB2AToB+A+g6ghI+xo8fry+//FKzZ8+W0+nUZZddpi1btshutwf0e2JiYjRnzpxWQ69ojXPlH86XefQb4Ynz5T/O2enZLMvQvXQAAADi3S4AAMAwwgcAADCK8AEAAIwifAAAAKMIHwAAwKiwDh/l5eXq37+/YmNjlZWVpZ07d/7g9uvXr1dmZqZiY2N1ySWX6A9/+IOhSsODP+dr9erVuuaaa9SzZ0/17NlTOTk5pz2/nY2///s6obq6WjabTWPHjg1ugWg3+g7/0Hf4j/7jLJ31G6GCpLq62oqOjraeeOIJ6y9/+Yt11113WUlJSZbL5Wpz+zfffNOKjIy0Fi1aZL333nvWzJkzrW7dull79uwxXHlo+Hu+/vVf/9UqLy+33nnnHev999+3br/9disxMdH661//arjy0PD3fJ2wf/9+60c/+pF1zTXXWPn5+WaKhV/oO/xD3+E/+o+zF7bhY9iwYVZhYaH38/Hjx63U1FSrtLS0ze1vvvlmKy8vz6ctKyvLuvvuu4NaZ7jw93yd7Ntvv7V69OhhrV27NlglhpX2nK9vv/3Wuuqqq6zHH3/cmjhxYpfvPMIVfYd/6Dv8R/9x9sLyssvRo0dVX1+vnJwcb1tERIRycnJUW1vb5j61tbU+20tSbm7uKbfvTNpzvk72zTff6NixY+rVq1ewygwb7T1fJSUl6tOnjyZNmmSiTLQDfYd/6Dv8R/8RGEF5vPrZ+tvf/qbjx4+3eqyy3W7XBx980OY+Tqezze2dTmfQ6gwX7TlfJ3vggQeUmpraqhPujNpzvrZt26Y1a9Zo9+7dBipEe9F3+Ie+w3/0H4ERluEDZi1YsEDV1dV6/fXXFRsbG+pyws7hw4f1i1/8QqtXr9a5554b6nKAsEHfcXr0H20Ly/Bx7rnnKjIyUi6Xy6fd5XLJ4XC0uY/D4fBr+86kPefrhEceeUQLFizQK6+8osGDBwezzLDh7/n6+OOPdeDAAY0ZM8bb1tLSIkmKiorS3r17dd555wW3aJwR+g7/0Hf4j/4jMMJyzkd0dLSGDBmimpoab1tLS4tqamqUnZ3d5j7Z2dk+20vSyy+/fMrtO5P2nC9JWrRokR566CFt2bJFQ4cONVFqWPD3fGVmZmrPnj3avXu3d/nZz36m66+/Xrt371ZaWprJ8vED6Dv8Q9/hP/qPAAn1jNdTqa6utmJiYqwnn3zSeu+996zJkydbSUlJltPptCzLsn7xi19YM2bM8G7/5ptvWlFRUdYjjzxivf/++9acOXO63O1y/pyvBQsWWNHR0dbvf/976/PPP/cuhw8fDtVPMMrf83UyZquHL/oO/9B3+I/+4+yFbfiwLMtatmyZlZ6ebkVHR1vDhg2zduzY4V33k5/8xJo4caLP9s8++6x1wQUXWNHR0dZFF11kbd682XDFoeXP+erXr58lqdUyZ84c84WHiL//+/o+Oo/wRt/hH/oO/9F/nB2bZVlWqEZdAABA1xOWcz4AAEDnRfgAAABGET4AAIBRhA8AAGAU4QMAABhF+AAAAEYRPgAAgFGEDwAAYBThAwAAGEX4AAAARhE+AACAUf8PWU99ZMFavRAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "df['gpt4_difference'] = abs(df['gpt4_en_kiwi'] - df['gpt4_en_xcomet'])\n",
    "df['alma_difference'] = abs(df['alma_en_kiwi'] - df['alma_en_xcomet'])\n",
    "# df['fluency_difference'] = abs(df['gpt4_en_fluency'] - df['alma_en_fluency'])\n",
    "\n",
    "mean_diff_kiwi = df['gpt4_difference'].mean()\n",
    "mean_diff_xcomet = df['alma_difference'].mean()\n",
    "# mean_diff_fluency = df['fluency_difference'].mean()\n",
    "\n",
    "# Оценка: если разница незначительна (например, < 0.01), можно брать среднее\n",
    "threshold = 0.01 \n",
    "\n",
    "print(mean_diff_kiwi)\n",
    "print(mean_diff_xcomet)\n",
    "# print(mean_diff_fluency)\n",
    "\n",
    "if mean_diff_kiwi < threshold and mean_diff_xcomet < threshold and mean_diff_fluency < threshold:\n",
    "    print(\"OK\")\n",
    "else:\n",
    "    print(\"not OK\")\n",
    "    \n",
    "    \n",
    "fig, (ax_a, ax_g) = plt.subplots(1, 2)\n",
    "ax_a.hist(df[\"alma_difference\"], bins=100)\n",
    "ax_a.set_title(\"Alma\")\n",
    "ax_g.hist(df[\"gpt4_difference\"], bins=100)\n",
    "ax_g.set_title(\"GPT4\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64476808-fcea-4457-a443-2f957e59bfcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DataSphere Kernel",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}