TrainMOCPO.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e1818712-0214-4e40-b391-097e21878162",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T17:34:26.239212Z",
     "iopub.status.busy": "2024-10-17T17:34:26.238874Z",
     "iopub.status.idle": "2024-10-17T17:34:29.151876Z",
     "shell.execute_reply": "2024-10-17T17:34:29.151177Z",
     "shell.execute_reply.started": "2024-10-17T17:34:26.239192Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "856d5abb8042446e9d12acc03ef2d824",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "wandb: Currently logged in as: sana3696g-wandb (sana3696g-wandb-hse). Use `wandb login --relogin` to force relogin\n",
      "wandb: WARNING If you're specifying your api key in code, ensure this code is not shared publicly.\n",
      "wandb: WARNING Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
      "wandb: Appending key for api.wandb.ai to your netrc file: /home/jupyter/.netrc\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()\n",
    "import wandb\n",
    "wandb.login(key=\"your_wandb_key\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1c09ec67-b1f3-4ee4-8cf8-0ed9f45a8e0c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T17:34:34.448685Z",
     "iopub.status.busy": "2024-10-17T17:34:34.448201Z",
     "iopub.status.idle": "2024-10-17T17:34:34.467412Z",
     "shell.execute_reply": "2024-10-17T17:34:34.466645Z",
     "shell.execute_reply.started": "2024-10-17T17:34:34.448661Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9046aff8-5436-49dc-8ea7-1711c4943062",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T17:34:35.457148Z",
     "iopub.status.busy": "2024-10-17T17:34:35.456556Z",
     "iopub.status.idle": "2024-10-17T17:34:35.495723Z",
     "shell.execute_reply": "2024-10-17T17:34:35.494856Z",
     "shell.execute_reply.started": "2024-10-17T17:34:35.457126Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jupyter/work/resources\n"
     ]
    }
   ],
   "source": [
    "%cd /home/jupyter/datasphere/project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e4a32b8b-ba23-496e-bc2d-6f9d871d210d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T17:34:37.169393Z",
     "iopub.status.busy": "2024-10-17T17:34:37.168985Z",
     "iopub.status.idle": "2024-10-17T17:34:45.255792Z",
     "shell.execute_reply": "2024-10-17T17:34:45.254774Z",
     "shell.execute_reply.started": "2024-10-17T17:34:37.169372Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%pip install --upgrade torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 -q\n",
    "%pip install -U tokenizers -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f482b809-1663-44fa-a16c-510439c5cb4b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T17:34:45.257355Z",
     "iopub.status.busy": "2024-10-17T17:34:45.256947Z",
     "iopub.status.idle": "2024-10-17T17:34:47.639851Z",
     "shell.execute_reply": "2024-10-17T17:34:47.639075Z",
     "shell.execute_reply.started": "2024-10-17T17:34:45.257331Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%pip install wandb -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bf794170-9cfd-46f5-b4ca-2d8cb19f7fb6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T17:34:47.641040Z",
     "iopub.status.busy": "2024-10-17T17:34:47.640670Z",
     "iopub.status.idle": "2024-10-17T17:35:08.561441Z",
     "shell.execute_reply": "2024-10-17T17:35:08.560658Z",
     "shell.execute_reply.started": "2024-10-17T17:34:47.641017Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33m  WARNING: The script transformers-cli is installed in '/home/jupyter/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "unbabel-comet 2.2.2 requires protobuf<5.0.0,>=4.24.4, but you have protobuf 3.20.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%pip install peft accelerate==0.27.2 transformers==4.38.2 protobuf==3.20 bitsandbytes sentencepiece sacrebleu ipython datasets evaluate deepspeed einops  wandb zstandard jsonlines trl==0.7.4 tokenizers -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "afb9a029-3bba-4529-a7e9-4d4687a07e90",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T17:35:08.563378Z",
     "iopub.status.busy": "2024-10-17T17:35:08.562966Z",
     "iopub.status.idle": "2024-10-17T17:35:08.575553Z",
     "shell.execute_reply": "2024-10-17T17:35:08.574876Z",
     "shell.execute_reply.started": "2024-10-17T17:35:08.563343Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jupyter/work/resources/modpo\n"
     ]
    }
   ],
   "source": [
    "%cd /home/jupyter/datasphere/project/modpo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4320af7d-55bc-4954-8c67-fad0d45ccbf5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T17:35:08.576553Z",
     "iopub.status.busy": "2024-10-17T17:35:08.576180Z",
     "iopub.status.idle": "2024-10-17T17:35:11.223201Z",
     "shell.execute_reply": "2024-10-17T17:35:11.222346Z",
     "shell.execute_reply.started": "2024-10-17T17:35:08.576519Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -okenizers (/home/jupyter/.local/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%pip install --upgrade tyro -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "318b15e7-313d-4552-b809-ed14f3ee59b2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 870/5424 [16:36<1:00:21,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.473, 'grad_norm': 3.1662542819976807, 'learning_rate': 9.386996802626953e-06, 'rewards/margins': 4.558123588562012, 'rewards/chosen': -4.838982582092285, 'rewards/rejected': -9.397107124328613, 'logps/margins': 21.54386329650879, 'logps/chosen': -10.284564018249512, 'logps/rejected': -31.828426361083984, 'accuracy': 0.8999999761581421, 'epoch': 0.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 880/5424 [16:44<1:00:12,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.1888, 'grad_norm': 3.029758930206299, 'learning_rate': 9.37302911193358e-06, 'rewards/margins': -0.22357389330863953, 'rewards/chosen': -5.452483177185059, 'rewards/rejected': -5.228909492492676, 'logps/margins': 2.2782464027404785, 'logps/chosen': -12.559453964233398, 'logps/rejected': -14.837699890136719, 'accuracy': 0.5, 'epoch': 0.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▋        | 890/5424 [16:53<1:02:10,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.4083, 'grad_norm': 2.8113043308258057, 'learning_rate': 9.358914711971607e-06, 'rewards/margins': 3.0816848278045654, 'rewards/chosen': -4.286379814147949, 'rewards/rejected': -7.368063926696777, 'logps/margins': 14.656011581420898, 'logps/chosen': -8.075750350952148, 'logps/rejected': -22.731761932373047, 'accuracy': 0.699999988079071, 'epoch': 0.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 900/5424 [17:01<1:01:24,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.3574, 'grad_norm': 3.009812116622925, 'learning_rate': 9.344654076260248e-06, 'rewards/margins': 0.7821928858757019, 'rewards/chosen': -6.05130672454834, 'rewards/rejected': -6.833499908447266, 'logps/margins': 5.113702297210693, 'logps/chosen': -15.370318412780762, 'logps/rejected': -20.484020233154297, 'accuracy': 0.6000000238418579, 'epoch': 0.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 910/5424 [17:09<1:01:07,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.2501, 'grad_norm': 2.586763381958008, 'learning_rate': 9.330247683224725e-06, 'rewards/margins': -0.9061592221260071, 'rewards/chosen': -6.649869441986084, 'rewards/rejected': -5.743709564208984, 'logps/margins': -2.500889778137207, 'logps/chosen': -18.000093460083008, 'logps/rejected': -15.4992036819458, 'accuracy': 0.6000000238418579, 'epoch': 0.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 920/5424 [17:17<1:01:26,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.3246, 'grad_norm': 3.703418731689453, 'learning_rate': 9.315696016180227e-06, 'rewards/margins': 3.7867062091827393, 'rewards/chosen': -4.757846832275391, 'rewards/rejected': -8.54455280303955, 'logps/margins': 19.485801696777344, 'logps/chosen': -10.17595386505127, 'logps/rejected': -29.661754608154297, 'accuracy': 0.8999999761581421, 'epoch': 0.51}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 930/5424 [17:26<1:00:22,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.0081, 'grad_norm': 2.1086173057556152, 'learning_rate': 9.300999563315689e-06, 'rewards/margins': 0.8038409948348999, 'rewards/chosen': -5.693100929260254, 'rewards/rejected': -6.496942043304443, 'logps/margins': 3.487497329711914, 'logps/chosen': -14.789260864257812, 'logps/rejected': -18.276758193969727, 'accuracy': 0.5, 'epoch': 0.51}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 940/5424 [17:34<1:02:35,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.859, 'grad_norm': 3.006026029586792, 'learning_rate': 9.286158817677417e-06, 'rewards/margins': 1.0101678371429443, 'rewards/chosen': -5.140589714050293, 'rewards/rejected': -6.150757789611816, 'logps/margins': 6.9351701736450195, 'logps/chosen': -11.973729133605957, 'logps/rejected': -18.908899307250977, 'accuracy': 0.699999988079071, 'epoch': 0.52}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 943/5424 [17:36<1:01:09,  1.22it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 18%|█▊        | 950/5424 [17:44<1:06:07,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.9874, 'grad_norm': 3.1475250720977783, 'learning_rate': 9.271174277152549e-06, 'rewards/margins': 1.7729727029800415, 'rewards/chosen': -5.211424827575684, 'rewards/rejected': -6.984397888183594, 'logps/margins': 9.925155639648438, 'logps/chosen': -11.086764335632324, 'logps/rejected': -21.011920928955078, 'accuracy': 0.6000000238418579, 'epoch': 0.53}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 960/5424 [17:52<59:26,  1.25it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.4904, 'grad_norm': 2.1135671138763428, 'learning_rate': 9.256046444452343e-06, 'rewards/margins': -1.0574671030044556, 'rewards/chosen': -5.309490203857422, 'rewards/rejected': -4.252023220062256, 'logps/margins': -1.1943809986114502, 'logps/chosen': -12.57465934753418, 'logps/rejected': -11.380278587341309, 'accuracy': 0.30000001192092896, 'epoch': 0.53}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 970/5424 [18:00<59:40,  1.24it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.7443, 'grad_norm': 2.453366994857788, 'learning_rate': 9.240775827095323e-06, 'rewards/margins': 0.8845325708389282, 'rewards/chosen': -5.200835227966309, 'rewards/rejected': -6.0853681564331055, 'logps/margins': 5.554448127746582, 'logps/chosen': -12.451799392700195, 'logps/rejected': -18.006244659423828, 'accuracy': 0.5, 'epoch': 0.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 980/5424 [18:08<59:34,  1.24it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.056, 'grad_norm': 2.6927661895751953, 'learning_rate': 9.225362937390244e-06, 'rewards/margins': 3.390026807785034, 'rewards/chosen': -5.522336483001709, 'rewards/rejected': -8.912363052368164, 'logps/margins': 14.193222045898438, 'logps/chosen': -14.46190071105957, 'logps/rejected': -28.655120849609375, 'accuracy': 0.6000000238418579, 'epoch': 0.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 982/5424 [18:10<59:26,  1.25it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 18%|█▊        | 990/5424 [18:19<1:04:24,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.568, 'grad_norm': 2.965471029281616, 'learning_rate': 9.209808292418909e-06, 'rewards/margins': 2.603837490081787, 'rewards/chosen': -6.177069664001465, 'rewards/rejected': -8.780906677246094, 'logps/margins': 14.811772346496582, 'logps/chosen': -14.922475814819336, 'logps/rejected': -29.734249114990234, 'accuracy': 0.699999988079071, 'epoch': 0.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 992/5424 [18:20<1:01:49,  1.19it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 18%|█▊        | 1000/5424 [18:29<1:05:11,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.1125, 'grad_norm': 3.0373191833496094, 'learning_rate': 9.194112414018819e-06, 'rewards/margins': 1.2550041675567627, 'rewards/chosen': -6.030290126800537, 'rewards/rejected': -7.285294532775879, 'logps/margins': 8.135400772094727, 'logps/chosen': -16.069257736206055, 'logps/rejected': -24.20465660095215, 'accuracy': 0.699999988079071, 'epoch': 0.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▊        | 1006/5424 [18:34<1:00:14,  1.22it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 19%|█▊        | 1008/5424 [18:38<1:35:46,  1.30s/it]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 19%|█▊        | 1010/5424 [18:42<1:57:10,  1.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.4422, 'grad_norm': 2.207125663757324, 'learning_rate': 9.17827582876567e-06, 'rewards/margins': 1.7611629962921143, 'rewards/chosen': -5.7919511795043945, 'rewards/rejected': -7.553114414215088, 'logps/margins': 12.388916015625, 'logps/chosen': -13.533502578735352, 'logps/rejected': -25.92241859436035, 'accuracy': 0.800000011920929, 'epoch': 0.56}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 1020/5424 [18:50<1:00:36,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 24.486, 'grad_norm': 4.518568992614746, 'learning_rate': 9.16229906795568e-06, 'rewards/margins': -2.318286418914795, 'rewards/chosen': -7.002549648284912, 'rewards/rejected': -4.684263706207275, 'logps/margins': -9.476079940795898, 'logps/chosen': -21.528762817382812, 'logps/rejected': -12.052680015563965, 'accuracy': 0.4000000059604645, 'epoch': 0.56}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 1028/5424 [18:57<1:00:11,  1.22it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 19%|█▉        | 1030/5424 [19:01<1:35:29,  1.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.7746, 'grad_norm': 1.3616280555725098, 'learning_rate': 9.146182667587774e-06, 'rewards/margins': 1.8513389825820923, 'rewards/chosen': -5.330580711364746, 'rewards/rejected': -7.181920051574707, 'logps/margins': 9.543803215026855, 'logps/chosen': -12.46239948272705, 'logps/rejected': -22.006200790405273, 'accuracy': 0.800000011920929, 'epoch': 0.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 1040/5424 [19:09<59:50,  1.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.9427, 'grad_norm': 4.535550594329834, 'learning_rate': 9.129927168345594e-06, 'rewards/margins': -0.23780092597007751, 'rewards/chosen': -4.79489803314209, 'rewards/rejected': -4.557097434997559, 'logps/margins': 0.32752180099487305, 'logps/chosen': -11.480095863342285, 'logps/rejected': -11.8076171875, 'accuracy': 0.20000000298023224, 'epoch': 0.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 1043/5424 [19:11<58:41,  1.24it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 19%|█▉        | 1050/5424 [19:19<1:03:39,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.1756, 'grad_norm': 3.1051528453826904, 'learning_rate': 9.113533115579362e-06, 'rewards/margins': 0.025220001116394997, 'rewards/chosen': -4.745850563049316, 'rewards/rejected': -4.771070957183838, 'logps/margins': 1.3081051111221313, 'logps/chosen': -11.961856842041016, 'logps/rejected': -13.2699613571167, 'accuracy': 0.4000000059604645, 'epoch': 0.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|█▉        | 1060/5424 [19:27<59:11,  1.23it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.7214, 'grad_norm': 3.020291566848755, 'learning_rate': 9.097001059287589e-06, 'rewards/margins': 0.9607834815979004, 'rewards/chosen': -5.4879469871521, 'rewards/rejected': -6.448729515075684, 'logps/margins': 4.153564453125, 'logps/chosen': -15.295354843139648, 'logps/rejected': -19.448917388916016, 'accuracy': 0.5, 'epoch': 0.59}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|█▉        | 1070/5424 [19:35<58:23,  1.24it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.7598, 'grad_norm': 3.271319627761841, 'learning_rate': 9.080331554098617e-06, 'rewards/margins': 2.601086139678955, 'rewards/chosen': -5.300355911254883, 'rewards/rejected': -7.901442527770996, 'logps/margins': 12.749253273010254, 'logps/chosen': -12.235665321350098, 'logps/rejected': -24.98491859436035, 'accuracy': 0.800000011920929, 'epoch': 0.59}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|█▉        | 1080/5424 [19:44<1:00:25,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.4121, 'grad_norm': 6.115197658538818, 'learning_rate': 9.063525159252013e-06, 'rewards/margins': 0.9802240133285522, 'rewards/chosen': -6.451292514801025, 'rewards/rejected': -7.431516170501709, 'logps/margins': 7.478054046630859, 'logps/chosen': -16.27468490600586, 'logps/rejected': -23.752737045288086, 'accuracy': 0.5, 'epoch': 0.6}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 1090/5424 [19:52<1:00:37,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.5972, 'grad_norm': 3.738187551498413, 'learning_rate': 9.046582438579814e-06, 'rewards/margins': 2.2707369327545166, 'rewards/chosen': -4.670331001281738, 'rewards/rejected': -6.941068172454834, 'logps/margins': 11.46412181854248, 'logps/chosen': -11.006529808044434, 'logps/rejected': -22.470651626586914, 'accuracy': 0.6000000238418579, 'epoch': 0.6}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 1100/5424 [20:00<1:00:06,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.9423, 'grad_norm': 2.021045684814453, 'learning_rate': 9.029503960487598e-06, 'rewards/margins': -0.35596245527267456, 'rewards/chosen': -6.048272132873535, 'rewards/rejected': -5.692309379577637, 'logps/margins': -2.3552803993225098, 'logps/chosen': -18.43783950805664, 'logps/rejected': -16.08255958557129, 'accuracy': 0.30000001192092896, 'epoch': 0.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 1110/5424 [20:08<59:32,  1.21it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.4169, 'grad_norm': 1.5971754789352417, 'learning_rate': 9.01229029793543e-06, 'rewards/margins': 1.2566192150115967, 'rewards/chosen': -5.872616767883301, 'rewards/rejected': -7.129236698150635, 'logps/margins': 4.625457763671875, 'logps/chosen': -16.038768768310547, 'logps/rejected': -20.664226531982422, 'accuracy': 0.6000000238418579, 'epoch': 0.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 1120/5424 [20:17<57:50,  1.24it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.5509, 'grad_norm': 1.6818255186080933, 'learning_rate': 8.994942028418624e-06, 'rewards/margins': -0.09500329196453094, 'rewards/chosen': -6.019637107849121, 'rewards/rejected': -5.924633979797363, 'logps/margins': 2.606217384338379, 'logps/chosen': -15.272439956665039, 'logps/rejected': -17.8786563873291, 'accuracy': 0.4000000059604645, 'epoch': 0.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 1127/5424 [20:22<57:33,  1.24it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 21%|██        | 1130/5424 [20:27<1:24:15,  1.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.7514, 'grad_norm': 2.33463716506958, 'learning_rate': 8.977459733948388e-06, 'rewards/margins': -0.6027255058288574, 'rewards/chosen': -5.383510589599609, 'rewards/rejected': -4.7807841300964355, 'logps/margins': -2.636470079421997, 'logps/chosen': -15.581108093261719, 'logps/rejected': -12.944636344909668, 'accuracy': 0.20000000298023224, 'epoch': 0.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 1140/5424 [20:35<59:04,  1.21it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.1697, 'grad_norm': 3.7011661529541016, 'learning_rate': 8.959844001032281e-06, 'rewards/margins': -0.8835283517837524, 'rewards/chosen': -5.6458539962768555, 'rewards/rejected': -4.762325763702393, 'logps/margins': -5.248634338378906, 'logps/chosen': -17.685123443603516, 'logps/rejected': -12.436487197875977, 'accuracy': 0.30000001192092896, 'epoch': 0.63}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 1141/5424 [20:36<1:01:19,  1.16it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 21%|██        | 1150/5424 [20:46<1:01:51,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.8839, 'grad_norm': 7.688358306884766, 'learning_rate': 8.942095420654542e-06, 'rewards/margins': 0.3737889230251312, 'rewards/chosen': -6.347555637359619, 'rewards/rejected': -6.721344947814941, 'logps/margins': 3.5358383655548096, 'logps/chosen': -15.095263481140137, 'logps/rejected': -18.631099700927734, 'accuracy': 0.5, 'epoch': 0.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██▏       | 1160/5424 [20:54<57:31,  1.24it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.0437, 'grad_norm': 2.6147141456604004, 'learning_rate': 8.92421458825627e-06, 'rewards/margins': 1.5428117513656616, 'rewards/chosen': -5.9360270500183105, 'rewards/rejected': -7.478838920593262, 'logps/margins': 7.2517805099487305, 'logps/chosen': -14.716207504272461, 'logps/rejected': -21.967987060546875, 'accuracy': 0.4000000059604645, 'epoch': 0.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 1170/5424 [21:02<57:35,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 23.0717, 'grad_norm': 3.7232813835144043, 'learning_rate': 8.906202103715435e-06, 'rewards/margins': -0.6146513223648071, 'rewards/chosen': -7.258846282958984, 'rewards/rejected': -6.644194602966309, 'logps/margins': -3.8527653217315674, 'logps/chosen': -21.07630729675293, 'logps/rejected': -17.22354507446289, 'accuracy': 0.4000000059604645, 'epoch': 0.65}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 1180/5424 [21:10<57:04,  1.24it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.0913, 'grad_norm': 1.6779441833496094, 'learning_rate': 8.888058571326768e-06, 'rewards/margins': 3.1266250610351562, 'rewards/chosen': -4.837032318115234, 'rewards/rejected': -7.963656425476074, 'logps/margins': 11.039449691772461, 'logps/chosen': -14.199809074401855, 'logps/rejected': -25.2392578125, 'accuracy': 0.800000011920929, 'epoch': 0.65}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 1185/5424 [21:14<59:05,  1.20it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 22%|██▏       | 1190/5424 [21:21<1:12:24,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 21.4909, 'grad_norm': 7.468592166900635, 'learning_rate': 8.869784599781473e-06, 'rewards/margins': -0.024387359619140625, 'rewards/chosen': -6.946310997009277, 'rewards/rejected': -6.921923637390137, 'logps/margins': 1.172191858291626, 'logps/chosen': -19.425535202026367, 'logps/rejected': -20.597726821899414, 'accuracy': 0.6000000238418579, 'epoch': 0.66}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 22%|██▏       | 1191/5424 [21:24<1:52:17,  1.59s/it]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 22%|██▏       | 1200/5424 [21:33<1:01:35,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.6331, 'grad_norm': 2.9290659427642822, 'learning_rate': 8.851380802146815e-06, 'rewards/margins': 0.257041871547699, 'rewards/chosen': -4.4745941162109375, 'rewards/rejected': -4.731636047363281, 'logps/margins': 0.5259411931037903, 'logps/chosen': -11.66088581085205, 'logps/rejected': -12.186826705932617, 'accuracy': 0.5, 'epoch': 0.66}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 1210/5424 [21:41<56:15,  1.25it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.2385, 'grad_norm': 2.684422492980957, 'learning_rate': 8.832847795845553e-06, 'rewards/margins': 3.7151122093200684, 'rewards/chosen': -4.360088348388672, 'rewards/rejected': -8.075201034545898, 'logps/margins': 20.016529083251953, 'logps/chosen': -8.849452018737793, 'logps/rejected': -28.865985870361328, 'accuracy': 0.8999999761581421, 'epoch': 0.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 1220/5424 [21:50<58:39,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.714, 'grad_norm': 1.4016526937484741, 'learning_rate': 8.81418620263522e-06, 'rewards/margins': 1.0530459880828857, 'rewards/chosen': -5.3463921546936035, 'rewards/rejected': -6.399438381195068, 'logps/margins': 3.343541383743286, 'logps/chosen': -14.67756175994873, 'logps/rejected': -18.021102905273438, 'accuracy': 0.699999988079071, 'epoch': 0.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 1230/5424 [21:58<56:21,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.2235, 'grad_norm': 2.642141580581665, 'learning_rate': 8.79539664858727e-06, 'rewards/margins': 2.276280164718628, 'rewards/chosen': -5.054806709289551, 'rewards/rejected': -7.331086158752441, 'logps/margins': 13.162333488464355, 'logps/chosen': -10.511981010437012, 'logps/rejected': -23.674314498901367, 'accuracy': 0.6000000238418579, 'epoch': 0.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 1240/5424 [22:06<54:14,  1.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.4537, 'grad_norm': 3.7141683101654053, 'learning_rate': 8.77647976406607e-06, 'rewards/margins': 1.7941668033599854, 'rewards/chosen': -3.961411237716675, 'rewards/rejected': -5.75557804107666, 'logps/margins': 9.443946838378906, 'logps/chosen': -7.6880202293396, 'logps/rejected': -17.131967544555664, 'accuracy': 0.800000011920929, 'epoch': 0.69}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 1250/5424 [22:14<59:11,  1.18it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.4867, 'grad_norm': 2.606421709060669, 'learning_rate': 8.757436183707757e-06, 'rewards/margins': 3.32000732421875, 'rewards/chosen': -5.588563919067383, 'rewards/rejected': -8.908571243286133, 'logps/margins': 14.385258674621582, 'logps/chosen': -13.763238906860352, 'logps/rejected': -28.148494720458984, 'accuracy': 0.8999999761581421, 'epoch': 0.69}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 1257/5424 [22:20<55:14,  1.26it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 23%|██▎       | 1260/5424 [22:24<1:18:46,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.2816, 'grad_norm': 3.0853869915008545, 'learning_rate': 8.738266546398938e-06, 'rewards/margins': 0.8340288996696472, 'rewards/chosen': -4.265200614929199, 'rewards/rejected': -5.099229335784912, 'logps/margins': 6.340903282165527, 'logps/chosen': -8.540224075317383, 'logps/rejected': -14.881129264831543, 'accuracy': 0.6000000238418579, 'epoch': 0.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 1270/5424 [22:33<57:03,  1.21it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.6246, 'grad_norm': 2.0786588191986084, 'learning_rate': 8.718971495255266e-06, 'rewards/margins': 1.3525949716567993, 'rewards/chosen': -5.673240661621094, 'rewards/rejected': -7.0258355140686035, 'logps/margins': 10.054628372192383, 'logps/chosen': -12.867321968078613, 'logps/rejected': -22.921951293945312, 'accuracy': 0.6000000238418579, 'epoch': 0.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▎       | 1280/5424 [22:41<56:21,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.2511, 'grad_norm': 5.21488618850708, 'learning_rate': 8.69955167759986e-06, 'rewards/margins': 2.5757651329040527, 'rewards/chosen': -5.064113140106201, 'rewards/rejected': -7.639878749847412, 'logps/margins': 13.409831047058105, 'logps/chosen': -11.908590316772461, 'logps/rejected': -25.31842041015625, 'accuracy': 0.8999999761581421, 'epoch': 0.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▎       | 1283/5424 [22:43<54:39,  1.26it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 24%|██▍       | 1290/5424 [22:51<1:01:55,  1.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.2078, 'grad_norm': 6.232727527618408, 'learning_rate': 8.68000774494159e-06, 'rewards/margins': 3.1713290214538574, 'rewards/chosen': -5.343134880065918, 'rewards/rejected': -8.514463424682617, 'logps/margins': 15.168004035949707, 'logps/chosen': -14.24632740020752, 'logps/rejected': -29.41432762145996, 'accuracy': 0.699999988079071, 'epoch': 0.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 1300/5424 [23:00<59:00,  1.16it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 23.5739, 'grad_norm': 5.019966125488281, 'learning_rate': 8.66034035295321e-06, 'rewards/margins': -1.0455372333526611, 'rewards/chosen': -7.59337854385376, 'rewards/rejected': -6.547841548919678, 'logps/margins': -0.8170045614242554, 'logps/chosen': -21.266504287719727, 'logps/rejected': -20.449499130249023, 'accuracy': 0.30000001192092896, 'epoch': 0.72}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 1303/5424 [23:02<54:12,  1.27it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 24%|██▍       | 1310/5424 [23:10<1:02:01,  1.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 7.8441, 'grad_norm': 2.527146577835083, 'learning_rate': 8.640550161449379e-06, 'rewards/margins': 3.648789882659912, 'rewards/chosen': -4.04988431930542, 'rewards/rejected': -7.698674201965332, 'logps/margins': 17.487693786621094, 'logps/chosen': -7.669066429138184, 'logps/rejected': -25.156761169433594, 'accuracy': 0.8999999761581421, 'epoch': 0.72}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 1320/5424 [23:18<55:01,  1.24it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.087, 'grad_norm': 6.156239032745361, 'learning_rate': 8.620637834364509e-06, 'rewards/margins': -1.437868356704712, 'rewards/chosen': -6.392221927642822, 'rewards/rejected': -4.9543538093566895, 'logps/margins': -6.008435249328613, 'logps/chosen': -18.109622955322266, 'logps/rejected': -12.10118579864502, 'accuracy': 0.20000000298023224, 'epoch': 0.73}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▍       | 1330/5424 [23:26<56:18,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.0622, 'grad_norm': 3.382005453109741, 'learning_rate': 8.6006040397305e-06, 'rewards/margins': 1.2466243505477905, 'rewards/chosen': -5.478442668914795, 'rewards/rejected': -6.725067138671875, 'logps/margins': 8.659399032592773, 'logps/chosen': -12.080465316772461, 'logps/rejected': -20.739864349365234, 'accuracy': 0.6000000238418579, 'epoch': 0.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▍       | 1340/5424 [23:35<53:54,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.1151, 'grad_norm': 3.241384744644165, 'learning_rate': 8.580449449654324e-06, 'rewards/margins': 1.3816008567810059, 'rewards/chosen': -5.20657205581665, 'rewards/rejected': -6.588172912597656, 'logps/margins': 6.572465419769287, 'logps/chosen': -12.338499069213867, 'logps/rejected': -18.91096305847168, 'accuracy': 0.699999988079071, 'epoch': 0.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▍       | 1343/5424 [23:37<54:37,  1.25it/s]Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      " 25%|██▍       | 1350/5424 [23:45<1:00:35,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.6821, 'grad_norm': 2.583583354949951, 'learning_rate': 8.560174740295482e-06, 'rewards/margins': 3.4713027477264404, 'rewards/chosen': -5.185531139373779, 'rewards/rejected': -8.656834602355957, 'logps/margins': 15.601877212524414, 'logps/chosen': -12.249704360961914, 'logps/rejected': -27.851581573486328, 'accuracy': 0.800000011920929, 'epoch': 0.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 1356/5424 [23:50<54:20,  1.25it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating response...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Could not load library libnvrtc.so.12. Error: libnvrtc.so.12: cannot open shared object file: No such file or directory\n",
      "\n",
      "  0%|          | 0/201 [00:00<?, ?it/s]\u001b[A\n",
      "  1%|          | 2/201 [00:00<01:05,  3.03it/s]\u001b[A\n",
      "  1%|▏         | 3/201 [00:01<01:42,  1.93it/s]\u001b[A\n",
      "  2%|▏         | 4/201 [00:02<01:48,  1.82it/s]\u001b[A\n",
      "  2%|▏         | 5/201 [00:02<01:54,  1.71it/s]\u001b[A\n",
      "  3%|▎         | 6/201 [00:03<01:58,  1.64it/s]\u001b[A\n",
      "  3%|▎         | 7/201 [00:04<02:01,  1.60it/s]\u001b[A\n",
      "  4%|▍         | 8/201 [00:04<02:03,  1.56it/s]\u001b[A\n",
      "  4%|▍         | 9/201 [00:05<02:11,  1.46it/s]\u001b[A\n",
      "  5%|▍         | 10/201 [00:06<02:05,  1.52it/s]\u001b[A\n",
      "  5%|▌         | 11/201 [00:06<02:05,  1.52it/s]\u001b[A\n",
      "  6%|▌         | 12/201 [00:07<02:05,  1.51it/s]\u001b[A\n",
      "  6%|▋         | 13/201 [00:08<02:10,  1.44it/s]\u001b[A\n",
      "  7%|▋         | 14/201 [00:08<02:07,  1.46it/s]\u001b[A\n",
      "  7%|▋         | 15/201 [00:09<02:06,  1.47it/s]\u001b[A\n",
      "  8%|▊         | 16/201 [00:10<02:05,  1.48it/s]\u001b[A\n",
      "  8%|▊         | 17/201 [00:10<02:10,  1.41it/s]\u001b[A\n",
      "  9%|▉         | 18/201 [00:11<02:09,  1.42it/s]\u001b[A\n",
      "  9%|▉         | 19/201 [00:12<02:08,  1.42it/s]\u001b[A\n",
      " 10%|▉         | 20/201 [00:13<02:05,  1.44it/s]\u001b[A\n",
      " 10%|█         | 21/201 [00:13<02:09,  1.40it/s]\u001b[A\n",
      " 11%|█         | 22/201 [00:14<02:07,  1.41it/s]\u001b[A\n",
      " 11%|█▏        | 23/201 [00:15<02:11,  1.36it/s]\u001b[A\n",
      " 12%|█▏        | 24/201 [00:15<02:05,  1.41it/s]\u001b[A\n",
      " 12%|█▏        | 25/201 [00:16<02:03,  1.43it/s]\u001b[A\n",
      " 13%|█▎        | 26/201 [00:17<02:00,  1.46it/s]\u001b[A\n",
      " 13%|█▎        | 27/201 [00:17<01:58,  1.47it/s]\u001b[A\n",
      " 14%|█▍        | 28/201 [00:18<02:02,  1.41it/s]\u001b[A\n",
      " 14%|█▍        | 29/201 [00:19<02:00,  1.43it/s]\u001b[A\n",
      " 15%|█▍        | 30/201 [00:20<01:57,  1.45it/s]\u001b[A\n",
      " 15%|█▌        | 31/201 [00:20<01:55,  1.48it/s]\u001b[A\n",
      " 16%|█▌        | 32/201 [00:21<01:52,  1.50it/s]\u001b[A\n",
      " 16%|█▋        | 33/201 [00:22<01:57,  1.43it/s]\u001b[A\n",
      " 17%|█▋        | 34/201 [00:22<01:55,  1.45it/s]\u001b[A\n",
      " 17%|█▋        | 35/201 [00:23<01:52,  1.48it/s]\u001b[A\n",
      " 18%|█▊        | 36/201 [00:24<01:51,  1.47it/s]\u001b[A\n",
      " 18%|█▊        | 37/201 [00:24<01:51,  1.48it/s]\u001b[A\n",
      " 19%|█▉        | 38/201 [00:25<01:48,  1.50it/s]\u001b[A\n",
      " 19%|█▉        | 39/201 [00:26<01:48,  1.49it/s]\u001b[A\n",
      " 20%|█▉        | 40/201 [00:26<01:47,  1.50it/s]\u001b[A\n",
      " 20%|██        | 41/201 [00:27<01:43,  1.54it/s]\u001b[A\n",
      " 21%|██        | 42/201 [00:28<01:49,  1.46it/s]\u001b[A\n",
      " 21%|██▏       | 43/201 [00:28<01:46,  1.48it/s]\u001b[A\n",
      " 22%|██▏       | 44/201 [00:29<01:45,  1.49it/s]\u001b[A\n",
      " 22%|██▏       | 45/201 [00:30<01:48,  1.43it/s]\u001b[A\n",
      " 23%|██▎       | 46/201 [00:30<01:46,  1.45it/s]\u001b[A\n",
      " 23%|██▎       | 47/201 [00:31<01:45,  1.47it/s]\u001b[A\n",
      " 24%|██▍       | 48/201 [00:32<01:40,  1.52it/s]\u001b[A\n",
      " 24%|██▍       | 49/201 [00:32<01:37,  1.56it/s]\u001b[A\n",
      " 25%|██▍       | 50/201 [00:33<01:37,  1.54it/s]\u001b[A\n",
      " 25%|██▌       | 51/201 [00:34<01:37,  1.54it/s]\u001b[A\n",
      " 26%|██▌       | 52/201 [00:34<01:37,  1.53it/s]\u001b[A\n",
      " 26%|██▋       | 53/201 [00:35<01:37,  1.53it/s]\u001b[A\n",
      " 27%|██▋       | 54/201 [00:36<01:41,  1.45it/s]\u001b[A\n",
      " 27%|██▋       | 55/201 [00:36<01:39,  1.47it/s]\u001b[A\n",
      " 28%|██▊       | 56/201 [00:37<01:37,  1.48it/s]\u001b[A\n",
      " 28%|██▊       | 57/201 [00:38<01:41,  1.41it/s]\u001b[A\n",
      " 29%|██▉       | 58/201 [00:38<01:40,  1.43it/s]\u001b[A\n",
      " 29%|██▉       | 59/201 [00:39<01:37,  1.46it/s]\u001b[A\n",
      " 30%|██▉       | 60/201 [00:40<01:35,  1.48it/s]\u001b[A\n",
      " 30%|███       | 61/201 [00:40<01:34,  1.49it/s]\u001b[A\n",
      " 31%|███       | 62/201 [00:41<01:33,  1.48it/s]\u001b[A\n",
      " 31%|███▏      | 63/201 [00:42<01:33,  1.48it/s]\u001b[A\n",
      " 32%|███▏      | 64/201 [00:42<01:32,  1.48it/s]\u001b[A\n",
      " 32%|███▏      | 65/201 [00:43<01:31,  1.48it/s]\u001b[A\n",
      " 33%|███▎      | 66/201 [00:44<01:35,  1.42it/s]\u001b[A\n",
      " 33%|███▎      | 67/201 [00:45<01:33,  1.43it/s]\u001b[A\n",
      " 34%|███▍      | 68/201 [00:45<01:31,  1.45it/s]\u001b[A\n",
      " 34%|███▍      | 69/201 [00:46<01:27,  1.51it/s]\u001b[A\n",
      " 35%|███▍      | 70/201 [00:47<01:26,  1.51it/s]\u001b[A\n",
      " 35%|███▌      | 71/201 [00:47<01:26,  1.50it/s]\u001b[A\n",
      " 36%|███▌      | 72/201 [00:48<01:25,  1.51it/s]\u001b[A\n",
      " 36%|███▋      | 73/201 [00:49<01:25,  1.51it/s]\u001b[A\n",
      " 37%|███▋      | 74/201 [00:49<01:23,  1.52it/s]\u001b[A\n",
      " 37%|███▋      | 75/201 [00:50<01:27,  1.43it/s]\u001b[A\n",
      " 38%|███▊      | 76/201 [00:51<01:26,  1.45it/s]\u001b[A\n",
      " 38%|███▊      | 77/201 [00:51<01:24,  1.47it/s]\u001b[A\n",
      " 39%|███▉      | 78/201 [00:52<01:27,  1.41it/s]\u001b[A\n",
      " 39%|███▉      | 79/201 [00:53<01:29,  1.37it/s]\u001b[A\n",
      " 40%|███▉      | 80/201 [00:54<01:29,  1.35it/s]\u001b[A\n",
      " 40%|████      | 81/201 [00:54<01:25,  1.40it/s]\u001b[A\n",
      " 41%|████      | 82/201 [00:55<01:27,  1.35it/s]\u001b[A\n",
      " 41%|████▏     | 83/201 [00:56<01:28,  1.33it/s]\u001b[A\n",
      " 42%|████▏     | 84/201 [00:56<01:24,  1.39it/s]\u001b[A\n",
      " 42%|████▏     | 85/201 [00:57<01:20,  1.43it/s]\u001b[A\n",
      " 43%|████▎     | 86/201 [00:58<01:18,  1.46it/s]\u001b[A\n",
      " 43%|████▎     | 87/201 [00:58<01:16,  1.49it/s]\u001b[A\n",
      " 44%|████▍     | 88/201 [00:59<01:15,  1.50it/s]\u001b[A\n",
      " 44%|████▍     | 89/201 [01:00<01:18,  1.42it/s]\u001b[A\n",
      " 45%|████▍     | 90/201 [01:01<01:16,  1.46it/s]\u001b[A\n",
      " 45%|████▌     | 91/201 [01:01<01:12,  1.51it/s]\u001b[A\n",
      " 46%|████▌     | 92/201 [01:02<01:15,  1.44it/s]\u001b[A\n",
      " 46%|████▋     | 93/201 [01:03<01:14,  1.46it/s]\u001b[A\n",
      " 47%|████▋     | 94/201 [01:03<01:12,  1.47it/s]\u001b[A\n",
      " 47%|████▋     | 95/201 [01:04<01:11,  1.48it/s]\u001b[A\n",
      " 48%|████▊     | 96/201 [01:04<01:08,  1.53it/s]\u001b[A\n",
      " 48%|████▊     | 97/201 [01:05<01:08,  1.53it/s]\u001b[A\n",
      " 49%|████▉     | 98/201 [01:06<01:07,  1.53it/s]\u001b[A\n",
      " 49%|████▉     | 99/201 [01:06<01:07,  1.52it/s]\u001b[A\n",
      " 50%|████▉     | 100/201 [01:07<01:06,  1.52it/s]\u001b[A\n",
      " 50%|█████     | 101/201 [01:08<01:05,  1.52it/s]\u001b[A\n",
      " 51%|█████     | 102/201 [01:08<01:04,  1.52it/s]\u001b[A\n",
      " 51%|█████     | 103/201 [01:09<01:04,  1.51it/s]\u001b[A\n",
      " 52%|█████▏    | 104/201 [01:10<01:05,  1.49it/s]\u001b[A\n",
      " 52%|█████▏    | 105/201 [01:10<01:04,  1.48it/s]\u001b[A\n",
      " 53%|█████▎    | 106/201 [01:11<01:03,  1.50it/s]\u001b[A\n",
      " 53%|█████▎    | 107/201 [01:12<01:02,  1.51it/s]\u001b[A\n",
      " 54%|█████▎    | 108/201 [01:12<01:01,  1.52it/s]\u001b[A\n",
      " 54%|█████▍    | 109/201 [01:13<01:00,  1.53it/s]\u001b[A\n",
      " 55%|█████▍    | 110/201 [01:14<00:58,  1.56it/s]\u001b[A\n",
      " 55%|█████▌    | 111/201 [01:14<00:58,  1.54it/s]\u001b[A\n",
      " 56%|█████▌    | 112/201 [01:15<00:58,  1.52it/s]\u001b[A\n",
      " 56%|█████▌    | 113/201 [01:16<00:57,  1.53it/s]\u001b[A\n",
      " 57%|█████▋    | 114/201 [01:16<00:56,  1.53it/s]\u001b[A\n",
      " 57%|█████▋    | 115/201 [01:17<00:57,  1.50it/s]\u001b[A\n",
      " 58%|█████▊    | 116/201 [01:18<00:56,  1.52it/s]\u001b[A\n",
      " 58%|█████▊    | 117/201 [01:18<00:55,  1.52it/s]\u001b[A\n",
      " 59%|█████▊    | 118/201 [01:19<00:54,  1.53it/s]\u001b[A\n",
      " 59%|█████▉    | 119/201 [01:20<00:53,  1.53it/s]\u001b[A\n",
      " 60%|█████▉    | 120/201 [01:20<00:53,  1.52it/s]\u001b[A\n",
      " 60%|██████    | 121/201 [01:21<00:52,  1.53it/s]\u001b[A\n",
      " 61%|██████    | 122/201 [01:22<00:52,  1.51it/s]\u001b[A\n",
      " 61%|██████    | 123/201 [01:22<00:53,  1.44it/s]\u001b[A\n",
      " 62%|██████▏   | 124/201 [01:23<00:52,  1.46it/s]\u001b[A\n",
      " 62%|██████▏   | 125/201 [01:24<00:50,  1.52it/s]\u001b[A\n",
      " 63%|██████▎   | 126/201 [01:24<00:48,  1.56it/s]\u001b[A\n",
      " 63%|██████▎   | 127/201 [01:25<00:47,  1.55it/s]\u001b[A\n",
      " 64%|██████▎   | 128/201 [01:26<00:47,  1.55it/s]\u001b[A\n",
      " 64%|██████▍   | 129/201 [01:26<00:46,  1.55it/s]\u001b[A\n",
      " 65%|██████▍   | 130/201 [01:27<00:46,  1.54it/s]\u001b[A\n",
      " 65%|██████▌   | 131/201 [01:28<00:45,  1.54it/s]\u001b[A\n",
      " 66%|██████▌   | 132/201 [01:29<01:00,  1.15it/s]\u001b[A\n",
      " 66%|██████▌   | 133/201 [01:30<00:55,  1.22it/s]\u001b[A\n",
      " 67%|██████▋   | 134/201 [01:30<00:51,  1.30it/s]\u001b[A\n",
      " 67%|██████▋   | 135/201 [01:31<00:48,  1.36it/s]\u001b[A\n",
      " 68%|██████▊   | 136/201 [01:32<00:46,  1.41it/s]\u001b[A\n",
      " 68%|██████▊   | 137/201 [01:32<00:44,  1.43it/s]\u001b[A\n",
      " 69%|██████▊   | 138/201 [01:33<00:43,  1.45it/s]\u001b[A\n",
      " 69%|██████▉   | 139/201 [01:34<00:41,  1.48it/s]\u001b[A\n",
      " 70%|██████▉   | 140/201 [01:34<00:41,  1.49it/s]\u001b[A\n",
      " 70%|███████   | 141/201 [01:35<00:40,  1.50it/s]\u001b[A\n",
      " 71%|███████   | 142/201 [01:36<00:41,  1.43it/s]\u001b[A\n",
      " 71%|███████   | 143/201 [01:36<00:40,  1.45it/s]\u001b[A\n",
      " 72%|███████▏  | 144/201 [01:37<00:39,  1.46it/s]\u001b[A\n",
      " 72%|███████▏  | 145/201 [01:38<00:37,  1.48it/s]\u001b[A\n",
      " 73%|███████▎  | 146/201 [01:38<00:38,  1.42it/s]\u001b[A\n",
      " 73%|███████▎  | 147/201 [01:40<00:49,  1.08it/s]\u001b[A\n",
      " 74%|███████▎  | 148/201 [01:40<00:44,  1.19it/s]\u001b[A\n",
      " 74%|███████▍  | 149/201 [01:41<00:41,  1.27it/s]\u001b[A\n",
      " 75%|███████▍  | 150/201 [01:42<00:38,  1.33it/s]\u001b[A\n",
      " 75%|███████▌  | 151/201 [01:43<00:38,  1.31it/s]\u001b[A\n",
      " 76%|███████▌  | 152/201 [01:43<00:36,  1.35it/s]\u001b[A\n",
      " 76%|███████▌  | 153/201 [01:44<00:34,  1.40it/s]\u001b[A\n",
      " 77%|███████▋  | 154/201 [01:45<00:32,  1.44it/s]\u001b[A\n",
      " 77%|███████▋  | 155/201 [01:45<00:31,  1.45it/s]\u001b[A\n",
      " 78%|███████▊  | 156/201 [01:46<00:30,  1.48it/s]\u001b[A\n",
      " 78%|███████▊  | 157/201 [01:47<00:30,  1.46it/s]\u001b[A\n",
      " 79%|███████▊  | 158/201 [01:47<00:29,  1.48it/s]\u001b[A\n",
      " 79%|███████▉  | 159/201 [01:48<00:27,  1.50it/s]\u001b[A\n",
      " 80%|███████▉  | 160/201 [01:49<00:27,  1.52it/s]\u001b[A\n",
      " 80%|████████  | 161/201 [01:49<00:26,  1.52it/s]\u001b[A\n",
      " 81%|████████  | 162/201 [01:50<00:25,  1.53it/s]\u001b[A\n",
      " 81%|████████  | 163/201 [01:51<00:24,  1.52it/s]\u001b[A\n",
      " 82%|████████▏ | 164/201 [01:51<00:24,  1.52it/s]\u001b[A\n",
      " 82%|████████▏ | 165/201 [01:52<00:24,  1.49it/s]\u001b[A\n",
      " 83%|████████▎ | 166/201 [01:53<00:23,  1.49it/s]\u001b[A\n",
      " 83%|████████▎ | 167/201 [01:53<00:22,  1.49it/s]\u001b[A\n",
      " 84%|████████▎ | 168/201 [01:55<00:29,  1.11it/s]\u001b[A\n",
      " 84%|████████▍ | 169/201 [01:55<00:26,  1.20it/s]\u001b[A\n",
      " 85%|████████▍ | 170/201 [01:56<00:24,  1.27it/s]\u001b[A\n",
      " 85%|████████▌ | 171/201 [01:57<00:22,  1.34it/s]\u001b[A\n",
      " 86%|████████▌ | 172/201 [01:57<00:21,  1.32it/s]\u001b[A\n",
      " 86%|████████▌ | 173/201 [01:58<00:20,  1.37it/s]\u001b[A\n",
      " 87%|████████▋ | 174/201 [01:59<00:19,  1.41it/s]\u001b[A\n",
      " 87%|████████▋ | 175/201 [01:59<00:18,  1.43it/s]\u001b[A\n",
      " 88%|████████▊ | 176/201 [02:00<00:17,  1.46it/s]\u001b[A\n",
      " 88%|████████▊ | 177/201 [02:01<00:16,  1.48it/s]\u001b[A\n",
      " 89%|████████▊ | 178/201 [02:01<00:15,  1.48it/s]\u001b[A\n",
      " 89%|████████▉ | 179/201 [02:02<00:14,  1.50it/s]\u001b[A\n",
      " 90%|████████▉ | 180/201 [02:03<00:13,  1.51it/s]\u001b[A\n",
      " 90%|█████████ | 181/201 [02:03<00:13,  1.52it/s]\u001b[A\n",
      " 91%|█████████ | 182/201 [02:04<00:12,  1.52it/s]\u001b[A\n",
      " 91%|█████████ | 183/201 [02:05<00:11,  1.52it/s]\u001b[A\n",
      " 92%|█████████▏| 184/201 [02:05<00:11,  1.51it/s]\u001b[A\n",
      " 92%|█████████▏| 185/201 [02:06<00:10,  1.52it/s]\u001b[A\n",
      " 93%|█████████▎| 186/201 [02:07<00:09,  1.51it/s]\u001b[A\n",
      " 93%|█████████▎| 187/201 [02:07<00:09,  1.50it/s]\u001b[A\n",
      " 94%|█████████▎| 188/201 [02:08<00:08,  1.50it/s]\u001b[A\n",
      " 94%|█████████▍| 189/201 [02:09<00:08,  1.50it/s]\u001b[A\n",
      " 95%|█████████▍| 190/201 [02:09<00:07,  1.50it/s]\u001b[A\n",
      " 95%|█████████▌| 191/201 [02:10<00:06,  1.51it/s]\u001b[A\n",
      " 96%|█████████▌| 192/201 [02:11<00:05,  1.52it/s]\u001b[A\n",
      " 96%|█████████▌| 193/201 [02:11<00:05,  1.51it/s]\u001b[A\n",
      " 97%|█████████▋| 194/201 [02:12<00:04,  1.55it/s]\u001b[A\n",
      " 97%|█████████▋| 195/201 [02:13<00:03,  1.54it/s]\u001b[A\n",
      " 98%|█████████▊| 196/201 [02:13<00:03,  1.53it/s]\u001b[A\n",
      " 98%|█████████▊| 197/201 [02:14<00:02,  1.52it/s]\u001b[A\n",
      " 99%|█████████▊| 198/201 [02:15<00:01,  1.53it/s]\u001b[A\n",
      " 99%|█████████▉| 199/201 [02:15<00:01,  1.53it/s]\u001b[A\n",
      "100%|█████████▉| 200/201 [02:16<00:00,  1.45it/s]\u001b[A\n",
      "                                                   A\n",
      " 25%|██▌       | 1356/5424 [30:44<54:20,  1.25it/s]\n",
      "100%|██████████| 201/201 [02:17<00:00,  1.48it/s]\u001b[A\n",
      "                                                 \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 14.920766830444336, 'eval_accuracy': 0.5970149253731343, 'eval_runtime': 413.8376, 'eval_samples_per_second': 0.486, 'eval_steps_per_second': 0.486, 'eval_rewards/margins': 1.232393741607666, 'eval_rewards/chosen': -5.593958377838135, 'eval_rewards/rejected': -6.826352119445801, 'eval_logps/margins': 7.617936134338379, 'eval_logps/chosen': -13.665102005004883, 'eval_logps/rejected': -21.283037185668945, 'epoch': 0.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 1360/5424 [30:58<50:39:54, 44.88s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.903, 'grad_norm': 9.110124588012695, 'learning_rate': 8.539780591843309e-06, 'rewards/margins': 4.4786696434021, 'rewards/chosen': -5.1232147216796875, 'rewards/rejected': -9.601884841918945, 'logps/margins': 20.259153366088867, 'logps/chosen': -11.087152481079102, 'logps/rejected': -31.3463077545166, 'accuracy': 0.800000011920929, 'epoch': 0.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 1370/5424 [31:06<2:18:43,  2.05s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.1161, 'grad_norm': 3.17655873298645, 'learning_rate': 8.519267688494168e-06, 'rewards/margins': 3.104088068008423, 'rewards/chosen': -4.167645454406738, 'rewards/rejected': -7.27173376083374, 'logps/margins': 14.304605484008789, 'logps/chosen': -9.605125427246094, 'logps/rejected': -23.909732818603516, 'accuracy': 0.800000011920929, 'epoch': 0.76}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 1380/5424 [31:14<55:26,  1.22it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.8362, 'grad_norm': 1.7616126537322998, 'learning_rate': 8.498636718428486e-06, 'rewards/margins': -0.050345517694950104, 'rewards/chosen': -4.187081336975098, 'rewards/rejected': -4.136735439300537, 'logps/margins': 2.0632057189941406, 'logps/chosen': -8.67735481262207, 'logps/rejected': -10.740560531616211, 'accuracy': 0.5, 'epoch': 0.76}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▌       | 1390/5424 [31:22<55:13,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.6414, 'grad_norm': 3.4854140281677246, 'learning_rate': 8.477888373787672e-06, 'rewards/margins': 1.7302411794662476, 'rewards/chosen': -4.758293628692627, 'rewards/rejected': -6.488534450531006, 'logps/margins': 8.552671432495117, 'logps/chosen': -11.97294807434082, 'logps/rejected': -20.525619506835938, 'accuracy': 0.800000011920929, 'epoch': 0.77}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▌       | 1400/5424 [31:30<55:22,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.808, 'grad_norm': 3.652327299118042, 'learning_rate': 8.457023350650893e-06, 'rewards/margins': 2.89038348197937, 'rewards/chosen': -6.038345813751221, 'rewards/rejected': -8.928729057312012, 'logps/margins': 10.149651527404785, 'logps/chosen': -16.98001480102539, 'logps/rejected': -27.129669189453125, 'accuracy': 0.699999988079071, 'epoch': 0.77}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▌       | 1410/5424 [31:41<1:16:11,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.4265, 'grad_norm': 4.244677543640137, 'learning_rate': 8.436042349011728e-06, 'rewards/margins': 2.1532833576202393, 'rewards/chosen': -5.4640045166015625, 'rewards/rejected': -7.617287635803223, 'logps/margins': 8.9605073928833, 'logps/chosen': -13.670480728149414, 'logps/rejected': -22.630985260009766, 'accuracy': 0.699999988079071, 'epoch': 0.78}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▌       | 1420/5424 [31:49<52:54,  1.26it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.9716, 'grad_norm': 1.8299922943115234, 'learning_rate': 8.414946072754672e-06, 'rewards/margins': 0.6284887194633484, 'rewards/chosen': -4.76852560043335, 'rewards/rejected': -5.397014141082764, 'logps/margins': 6.5863823890686035, 'logps/chosen': -9.955560684204102, 'logps/rejected': -16.541942596435547, 'accuracy': 0.699999988079071, 'epoch': 0.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▋       | 1430/5424 [31:57<52:45,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.3402, 'grad_norm': 3.147454023361206, 'learning_rate': 8.393735229631536e-06, 'rewards/margins': 2.881399631500244, 'rewards/chosen': -4.124011993408203, 'rewards/rejected': -7.0054121017456055, 'logps/margins': 13.597552299499512, 'logps/chosen': -8.686369895935059, 'logps/rejected': -22.283924102783203, 'accuracy': 0.6000000238418579, 'epoch': 0.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 1440/5424 [32:05<53:03,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.1137, 'grad_norm': 1.9623808860778809, 'learning_rate': 8.37241053123769e-06, 'rewards/margins': 3.3831629753112793, 'rewards/chosen': -4.147034168243408, 'rewards/rejected': -7.530198097229004, 'logps/margins': 17.311887741088867, 'logps/chosen': -7.312302589416504, 'logps/rejected': -24.624191284179688, 'accuracy': 0.6000000238418579, 'epoch': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 1450/5424 [32:13<55:16,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.8272, 'grad_norm': 3.967428684234619, 'learning_rate': 8.3509726929882e-06, 'rewards/margins': 2.8563246726989746, 'rewards/chosen': -5.557080268859863, 'rewards/rejected': -8.41340446472168, 'logps/margins': 14.923563957214355, 'logps/chosen': -11.651995658874512, 'logps/rejected': -26.575557708740234, 'accuracy': 0.8999999761581421, 'epoch': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 1460/5424 [32:23<56:34,  1.17it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.4955, 'grad_norm': 3.320117235183716, 'learning_rate': 8.32942243409382e-06, 'rewards/margins': 2.3081417083740234, 'rewards/chosen': -5.254875183105469, 'rewards/rejected': -7.56301736831665, 'logps/margins': 12.987821578979492, 'logps/chosen': -9.996335983276367, 'logps/rejected': -22.984161376953125, 'accuracy': 0.800000011920929, 'epoch': 0.81}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 1470/5424 [32:32<55:10,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.5507, 'grad_norm': 10.182258605957031, 'learning_rate': 8.307760477536868e-06, 'rewards/margins': 0.8943980932235718, 'rewards/chosen': -6.249564170837402, 'rewards/rejected': -7.143962860107422, 'logps/margins': 4.795457363128662, 'logps/chosen': -16.918209075927734, 'logps/rejected': -21.713668823242188, 'accuracy': 0.5, 'epoch': 0.81}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 1480/5424 [32:42<1:00:19,  1.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.0405, 'grad_norm': 2.3264880180358887, 'learning_rate': 8.28598755004697e-06, 'rewards/margins': 2.226961135864258, 'rewards/chosen': -5.727206230163574, 'rewards/rejected': -7.954167366027832, 'logps/margins': 14.542486190795898, 'logps/chosen': -11.629855155944824, 'logps/rejected': -26.17234230041504, 'accuracy': 0.699999988079071, 'epoch': 0.82}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 1490/5424 [32:50<54:22,  1.21it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.6107, 'grad_norm': 2.137305974960327, 'learning_rate': 8.264104382076673e-06, 'rewards/margins': 3.389930009841919, 'rewards/chosen': -4.596156597137451, 'rewards/rejected': -7.986086368560791, 'logps/margins': 15.477956771850586, 'logps/chosen': -9.215790748596191, 'logps/rejected': -24.693748474121094, 'accuracy': 0.800000011920929, 'epoch': 0.82}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 1500/5424 [32:59<55:01,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.4937, 'grad_norm': 4.951788902282715, 'learning_rate': 8.24211170777695e-06, 'rewards/margins': 3.120495319366455, 'rewards/chosen': -6.619574546813965, 'rewards/rejected': -9.740071296691895, 'logps/margins': 17.91663360595703, 'logps/chosen': -16.714426040649414, 'logps/rejected': -34.63105773925781, 'accuracy': 0.800000011920929, 'epoch': 0.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 1510/5424 [33:07<51:43,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 22.1926, 'grad_norm': 1.508258581161499, 'learning_rate': 8.220010264972562e-06, 'rewards/margins': -0.759492039680481, 'rewards/chosen': -7.129822731018066, 'rewards/rejected': -6.370330333709717, 'logps/margins': -2.538806438446045, 'logps/chosen': -19.94478416442871, 'logps/rejected': -17.40597915649414, 'accuracy': 0.5, 'epoch': 0.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 1520/5424 [33:15<51:26,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.0523, 'grad_norm': 6.064990043640137, 'learning_rate': 8.197800795137307e-06, 'rewards/margins': 0.49749755859375, 'rewards/chosen': -5.095200538635254, 'rewards/rejected': -5.592698097229004, 'logps/margins': 2.444831609725952, 'logps/chosen': -13.394002914428711, 'logps/rejected': -15.838834762573242, 'accuracy': 0.5, 'epoch': 0.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 1530/5424 [33:23<52:00,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.7633, 'grad_norm': 1.9322243928909302, 'learning_rate': 8.175484043369147e-06, 'rewards/margins': 3.79650616645813, 'rewards/chosen': -5.646278381347656, 'rewards/rejected': -9.442785263061523, 'logps/margins': 20.101318359375, 'logps/chosen': -13.609617233276367, 'logps/rejected': -33.7109375, 'accuracy': 0.8999999761581421, 'epoch': 0.85}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 1540/5424 [33:31<52:08,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.0942, 'grad_norm': 4.564767360687256, 'learning_rate': 8.153060758365208e-06, 'rewards/margins': 1.0177675485610962, 'rewards/chosen': -6.03346586227417, 'rewards/rejected': -7.051232814788818, 'logps/margins': 5.61105489730835, 'logps/chosen': -14.9147310256958, 'logps/rejected': -20.52578353881836, 'accuracy': 0.4000000059604645, 'epoch': 0.85}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▊       | 1550/5424 [33:39<52:02,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.5597, 'grad_norm': 4.7600884437561035, 'learning_rate': 8.130531692396664e-06, 'rewards/margins': 1.1257097721099854, 'rewards/chosen': -6.680582523345947, 'rewards/rejected': -7.8062920570373535, 'logps/margins': 7.685629367828369, 'logps/chosen': -16.32986831665039, 'logps/rejected': -24.015499114990234, 'accuracy': 0.6000000238418579, 'epoch': 0.86}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 1560/5424 [33:47<52:36,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.0706, 'grad_norm': 6.956604957580566, 'learning_rate': 8.1078976012835e-06, 'rewards/margins': 2.6198906898498535, 'rewards/chosen': -6.580979824066162, 'rewards/rejected': -9.200871467590332, 'logps/margins': 13.630633354187012, 'logps/chosen': -17.796741485595703, 'logps/rejected': -31.4273738861084, 'accuracy': 0.6000000238418579, 'epoch': 0.86}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 1570/5424 [33:55<54:08,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.0389, 'grad_norm': 4.005314350128174, 'learning_rate': 8.085159244369146e-06, 'rewards/margins': 2.807217836380005, 'rewards/chosen': -6.483763217926025, 'rewards/rejected': -9.29098129272461, 'logps/margins': 12.379849433898926, 'logps/chosen': -17.53799057006836, 'logps/rejected': -29.917837142944336, 'accuracy': 0.699999988079071, 'epoch': 0.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 1580/5424 [34:04<54:11,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 23.8444, 'grad_norm': 4.980125904083252, 'learning_rate': 8.062317384495023e-06, 'rewards/margins': -1.1870554685592651, 'rewards/chosen': -7.9652252197265625, 'rewards/rejected': -6.778170108795166, 'logps/margins': -2.7348029613494873, 'logps/chosen': -21.28317642211914, 'logps/rejected': -18.54837417602539, 'accuracy': 0.4000000059604645, 'epoch': 0.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 1590/5424 [34:12<51:12,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.826, 'grad_norm': 0.0, 'learning_rate': 8.039372787974928e-06, 'rewards/margins': 0.7453154921531677, 'rewards/chosen': -3.926908493041992, 'rewards/rejected': -4.672224044799805, 'logps/margins': 2.918670177459717, 'logps/chosen': -9.098824501037598, 'logps/rejected': -12.017492294311523, 'accuracy': 0.5, 'epoch': 0.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 1600/5424 [34:20<50:46,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.3957, 'grad_norm': 5.1904473304748535, 'learning_rate': 8.016326224569338e-06, 'rewards/margins': -0.2908414304256439, 'rewards/chosen': -4.542311668395996, 'rewards/rejected': -4.251470565795898, 'logps/margins': -0.4982810914516449, 'logps/chosen': -11.010189056396484, 'logps/rejected': -10.511907577514648, 'accuracy': 0.4000000059604645, 'epoch': 0.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██▉       | 1610/5424 [34:30<1:04:45,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 26.4538, 'grad_norm': 1.856727957725525, 'learning_rate': 7.993178467459584e-06, 'rewards/margins': -1.496856927871704, 'rewards/chosen': -7.124918460845947, 'rewards/rejected': -5.628061771392822, 'logps/margins': -7.267629146575928, 'logps/chosen': -23.652490615844727, 'logps/rejected': -16.38486099243164, 'accuracy': 0.4000000059604645, 'epoch': 0.89}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██▉       | 1620/5424 [34:38<51:22,  1.23it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.3013, 'grad_norm': 3.527959108352661, 'learning_rate': 7.969930293221905e-06, 'rewards/margins': 2.833458662033081, 'rewards/chosen': -5.495981693267822, 'rewards/rejected': -8.329440116882324, 'logps/margins': 11.48737621307373, 'logps/chosen': -14.57396125793457, 'logps/rejected': -26.061336517333984, 'accuracy': 0.699999988079071, 'epoch': 0.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 1630/5424 [34:46<51:08,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.519, 'grad_norm': 1.4873405694961548, 'learning_rate': 7.946582481801404e-06, 'rewards/margins': 1.1601804494857788, 'rewards/chosen': -6.7667131423950195, 'rewards/rejected': -7.926894187927246, 'logps/margins': 6.154648780822754, 'logps/chosen': -17.582965850830078, 'logps/rejected': -23.737611770629883, 'accuracy': 0.5, 'epoch': 0.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 1640/5424 [34:54<50:12,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.2744, 'grad_norm': 4.202121734619141, 'learning_rate': 7.92313581648588e-06, 'rewards/margins': 1.8642938137054443, 'rewards/chosen': -4.1750993728637695, 'rewards/rejected': -6.039393424987793, 'logps/margins': 10.66430950164795, 'logps/chosen': -8.533129692077637, 'logps/rejected': -19.19744110107422, 'accuracy': 0.6000000238418579, 'epoch': 0.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 1650/5424 [35:02<52:13,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.3696, 'grad_norm': 7.409375190734863, 'learning_rate': 7.899591083879542e-06, 'rewards/margins': 2.1183743476867676, 'rewards/chosen': -5.262244701385498, 'rewards/rejected': -7.380619049072266, 'logps/margins': 5.960939884185791, 'logps/chosen': -15.503561019897461, 'logps/rejected': -21.464500427246094, 'accuracy': 0.699999988079071, 'epoch': 0.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███       | 1660/5424 [35:10<50:14,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.9755, 'grad_norm': 3.4077744483947754, 'learning_rate': 7.87594907387663e-06, 'rewards/margins': 0.52447509765625, 'rewards/chosen': -6.51010799407959, 'rewards/rejected': -7.034582614898682, 'logps/margins': 2.510389566421509, 'logps/chosen': -17.371156692504883, 'logps/rejected': -19.881547927856445, 'accuracy': 0.30000001192092896, 'epoch': 0.92}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███       | 1670/5424 [35:18<50:10,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.276, 'grad_norm': 2.2151193618774414, 'learning_rate': 7.852210579634912e-06, 'rewards/margins': 0.9393287897109985, 'rewards/chosen': -4.220452785491943, 'rewards/rejected': -5.159782409667969, 'logps/margins': 2.3603835105895996, 'logps/chosen': -11.346477508544922, 'logps/rejected': -13.70686149597168, 'accuracy': 0.6000000238418579, 'epoch': 0.92}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███       | 1680/5424 [35:26<49:13,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.2695, 'grad_norm': 1.9876716136932373, 'learning_rate': 7.828376397549063e-06, 'rewards/margins': -0.14661791920661926, 'rewards/chosen': -5.44817590713501, 'rewards/rejected': -5.301558494567871, 'logps/margins': -1.1658083200454712, 'logps/chosen': -14.503458023071289, 'logps/rejected': -13.33764934539795, 'accuracy': 0.5, 'epoch': 0.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███       | 1690/5424 [35:34<50:04,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.4793, 'grad_norm': 5.282959938049316, 'learning_rate': 7.804447327223969e-06, 'rewards/margins': 1.1291416883468628, 'rewards/chosen': -5.276439189910889, 'rewards/rejected': -6.405580997467041, 'logps/margins': 6.9524664878845215, 'logps/chosen': -12.510913848876953, 'logps/rejected': -19.46337890625, 'accuracy': 0.6000000238418579, 'epoch': 0.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███▏      | 1700/5424 [35:43<52:14,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.2195, 'grad_norm': 5.350795269012451, 'learning_rate': 7.78042417144788e-06, 'rewards/margins': 0.5749262571334839, 'rewards/chosen': -6.88068151473999, 'rewards/rejected': -7.4556074142456055, 'logps/margins': 3.8335978984832764, 'logps/chosen': -17.693500518798828, 'logps/rejected': -21.527099609375, 'accuracy': 0.6000000238418579, 'epoch': 0.94}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 1710/5424 [35:51<49:44,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.2364, 'grad_norm': 4.676078796386719, 'learning_rate': 7.756307736165496e-06, 'rewards/margins': 2.4061684608459473, 'rewards/chosen': -3.8523051738739014, 'rewards/rejected': -6.258473873138428, 'logps/margins': 10.64726448059082, 'logps/chosen': -7.7455244064331055, 'logps/rejected': -18.392789840698242, 'accuracy': 0.699999988079071, 'epoch': 0.95}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 1720/5424 [35:59<51:42,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.0958, 'grad_norm': 3.2915172576904297, 'learning_rate': 7.73209883045091e-06, 'rewards/margins': 1.0748727321624756, 'rewards/chosen': -6.011447429656982, 'rewards/rejected': -7.086320400238037, 'logps/margins': 6.7631354331970215, 'logps/chosen': -15.640615463256836, 'logps/rejected': -22.403751373291016, 'accuracy': 0.699999988079071, 'epoch': 0.95}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 1730/5424 [36:07<49:19,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.8056, 'grad_norm': 3.7821736335754395, 'learning_rate': 7.707798266480482e-06, 'rewards/margins': 3.140820026397705, 'rewards/chosen': -4.880075931549072, 'rewards/rejected': -8.020895957946777, 'logps/margins': 15.866643905639648, 'logps/chosen': -10.191213607788086, 'logps/rejected': -26.057857513427734, 'accuracy': 0.699999988079071, 'epoch': 0.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 1740/5424 [36:15<48:59,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.1652, 'grad_norm': 2.0831449031829834, 'learning_rate': 7.683406859505575e-06, 'rewards/margins': 1.5195001363754272, 'rewards/chosen': -4.7060723304748535, 'rewards/rejected': -6.225572109222412, 'logps/margins': 8.093459129333496, 'logps/chosen': -10.229117393493652, 'logps/rejected': -18.32257652282715, 'accuracy': 0.6000000238418579, 'epoch': 0.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 1750/5424 [36:23<47:58,  1.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.3863, 'grad_norm': 2.313776969909668, 'learning_rate': 7.658925427825224e-06, 'rewards/margins': 2.3043267726898193, 'rewards/chosen': -3.871371030807495, 'rewards/rejected': -6.1756978034973145, 'logps/margins': 12.169134140014648, 'logps/chosen': -7.892942905426025, 'logps/rejected': -20.062076568603516, 'accuracy': 0.800000011920929, 'epoch': 0.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 1760/5424 [36:33<51:44,  1.18it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.4865, 'grad_norm': 7.660449981689453, 'learning_rate': 7.634354792758664e-06, 'rewards/margins': 1.8762102127075195, 'rewards/chosen': -5.047272682189941, 'rewards/rejected': -6.923482418060303, 'logps/margins': 10.515466690063477, 'logps/chosen': -11.704118728637695, 'logps/rejected': -22.21958351135254, 'accuracy': 0.800000011920929, 'epoch': 0.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1770/5424 [36:41<50:38,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.5592, 'grad_norm': 5.149654865264893, 'learning_rate': 7.609695778617786e-06, 'rewards/margins': 0.18779902160167694, 'rewards/chosen': -6.601194858551025, 'rewards/rejected': -6.788993835449219, 'logps/margins': 0.9888935089111328, 'logps/chosen': -17.731849670410156, 'logps/rejected': -18.72074317932129, 'accuracy': 0.4000000059604645, 'epoch': 0.98}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1780/5424 [36:49<49:26,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.6215, 'grad_norm': 4.94338321685791, 'learning_rate': 7.584949212679481e-06, 'rewards/margins': 1.7907555103302002, 'rewards/chosen': -5.663502216339111, 'rewards/rejected': -7.454257011413574, 'logps/margins': 11.4433012008667, 'logps/chosen': -12.677032470703125, 'logps/rejected': -24.12033462524414, 'accuracy': 0.5, 'epoch': 0.98}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1790/5424 [37:02<1:35:27,  1.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.448, 'grad_norm': 2.9585659503936768, 'learning_rate': 7.5601159251578885e-06, 'rewards/margins': 2.2975497245788574, 'rewards/chosen': -5.646200656890869, 'rewards/rejected': -7.943750858306885, 'logps/margins': 12.825654983520508, 'logps/chosen': -12.438457489013672, 'logps/rejected': -25.264110565185547, 'accuracy': 0.6000000238418579, 'epoch': 0.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1800/5424 [37:12<56:59,  1.06it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 22.7962, 'grad_norm': 2.6053426265716553, 'learning_rate': 7.535196749176535e-06, 'rewards/margins': 0.6128777861595154, 'rewards/chosen': -7.749342441558838, 'rewards/rejected': -8.362221717834473, 'logps/margins': 5.273512363433838, 'logps/chosen': -21.05111312866211, 'logps/rejected': -26.324626922607422, 'accuracy': 0.5, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1810/5424 [37:22<59:23,  1.01it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.2414, 'grad_norm': 8.846335411071777, 'learning_rate': 7.510192520740396e-06, 'rewards/margins': -0.3581724166870117, 'rewards/chosen': -6.504356384277344, 'rewards/rejected': -6.14618444442749, 'logps/margins': -0.1433391571044922, 'logps/chosen': -16.918325424194336, 'logps/rejected': -16.774988174438477, 'accuracy': 0.4000000059604645, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▎      | 1820/5424 [37:30<49:02,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.4862, 'grad_norm': 4.5500969886779785, 'learning_rate': 7.485104078707837e-06, 'rewards/margins': 0.8850962519645691, 'rewards/chosen': -5.526177883148193, 'rewards/rejected': -6.411273956298828, 'logps/margins': 4.570897102355957, 'logps/chosen': -13.061601638793945, 'logps/rejected': -17.632497787475586, 'accuracy': 0.5, 'epoch': 1.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▎      | 1830/5424 [37:38<47:06,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.1319, 'grad_norm': 3.031832218170166, 'learning_rate': 7.459932264762481e-06, 'rewards/margins': -0.17033301293849945, 'rewards/chosen': -6.057919502258301, 'rewards/rejected': -5.8875861167907715, 'logps/margins': 0.31434792280197144, 'logps/chosen': -16.262617111206055, 'logps/rejected': -16.576967239379883, 'accuracy': 0.5, 'epoch': 1.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 1840/5424 [37:46<49:28,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.7749, 'grad_norm': 2.4659674167633057, 'learning_rate': 7.434677923384966e-06, 'rewards/margins': 0.32654276490211487, 'rewards/chosen': -6.411888122558594, 'rewards/rejected': -6.738431453704834, 'logps/margins': 3.152549982070923, 'logps/chosen': -18.39792823791504, 'logps/rejected': -21.550477981567383, 'accuracy': 0.4000000059604645, 'epoch': 1.02}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 1850/5424 [37:54<47:23,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.3146, 'grad_norm': 2.029222249984741, 'learning_rate': 7.4093419018246135e-06, 'rewards/margins': 1.4361984729766846, 'rewards/chosen': -4.3573479652404785, 'rewards/rejected': -5.793546199798584, 'logps/margins': 6.369390487670898, 'logps/chosen': -9.656126022338867, 'logps/rejected': -16.025516510009766, 'accuracy': 0.6000000238418579, 'epoch': 1.02}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 1860/5424 [38:02<48:12,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.6654, 'grad_norm': 4.148105144500732, 'learning_rate': 7.38392505007101e-06, 'rewards/margins': -0.6400356292724609, 'rewards/chosen': -5.896147727966309, 'rewards/rejected': -5.256112098693848, 'logps/margins': 1.552794098854065, 'logps/chosen': -13.550424575805664, 'logps/rejected': -15.103218078613281, 'accuracy': 0.30000001192092896, 'epoch': 1.03}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 1870/5424 [38:10<49:02,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.5003, 'grad_norm': 5.487306118011475, 'learning_rate': 7.3584282208254785e-06, 'rewards/margins': 1.7540098428726196, 'rewards/chosen': -4.881596088409424, 'rewards/rejected': -6.635605812072754, 'logps/margins': 8.023588180541992, 'logps/chosen': -10.11054801940918, 'logps/rejected': -18.13413429260254, 'accuracy': 0.699999988079071, 'epoch': 1.03}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▍      | 1880/5424 [38:18<46:34,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.1225, 'grad_norm': 3.134225606918335, 'learning_rate': 7.3328522694724895e-06, 'rewards/margins': 1.090826392173767, 'rewards/chosen': -5.007294654846191, 'rewards/rejected': -6.098121166229248, 'logps/margins': 2.2592031955718994, 'logps/chosen': -14.012784957885742, 'logps/rejected': -16.271987915039062, 'accuracy': 0.6000000238418579, 'epoch': 1.04}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▍      | 1890/5424 [38:26<46:59,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.0725, 'grad_norm': 7.365496635437012, 'learning_rate': 7.307198054050943e-06, 'rewards/margins': 0.31348317861557007, 'rewards/chosen': -3.9564051628112793, 'rewards/rejected': -4.269888877868652, 'logps/margins': 1.8709170818328857, 'logps/chosen': -8.106683731079102, 'logps/rejected': -9.977601051330566, 'accuracy': 0.5, 'epoch': 1.05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 1900/5424 [38:34<47:10,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.688, 'grad_norm': 3.82757306098938, 'learning_rate': 7.281466435225408e-06, 'rewards/margins': 1.218552589416504, 'rewards/chosen': -4.5628204345703125, 'rewards/rejected': -5.781373023986816, 'logps/margins': 7.3442864418029785, 'logps/chosen': -9.849299430847168, 'logps/rejected': -17.193584442138672, 'accuracy': 0.6000000238418579, 'epoch': 1.05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 1910/5424 [38:42<49:11,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 23.2132, 'grad_norm': 3.147826671600342, 'learning_rate': 7.255658276257221e-06, 'rewards/margins': -0.963045597076416, 'rewards/chosen': -7.296431064605713, 'rewards/rejected': -6.333385467529297, 'logps/margins': -2.7673540115356445, 'logps/chosen': -20.9105167388916, 'logps/rejected': -18.14316177368164, 'accuracy': 0.20000000298023224, 'epoch': 1.06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 1920/5424 [38:50<48:21,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.8223, 'grad_norm': 3.6452481746673584, 'learning_rate': 7.229774442975543e-06, 'rewards/margins': 4.465226173400879, 'rewards/chosen': -5.556630611419678, 'rewards/rejected': -10.021856307983398, 'logps/margins': 20.399728775024414, 'logps/chosen': -14.048835754394531, 'logps/rejected': -34.44856643676758, 'accuracy': 0.800000011920929, 'epoch': 1.06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 1930/5424 [38:59<48:18,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.0213, 'grad_norm': 3.603363037109375, 'learning_rate': 7.203815803748306e-06, 'rewards/margins': 3.1232330799102783, 'rewards/chosen': -5.167369842529297, 'rewards/rejected': -8.29060173034668, 'logps/margins': 15.735727310180664, 'logps/chosen': -12.34134578704834, 'logps/rejected': -28.077072143554688, 'accuracy': 0.699999988079071, 'epoch': 1.07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 1940/5424 [39:07<47:41,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.4593, 'grad_norm': 6.914211273193359, 'learning_rate': 7.177783229453086e-06, 'rewards/margins': 0.8834846615791321, 'rewards/chosen': -5.980111122131348, 'rewards/rejected': -6.863596439361572, 'logps/margins': 5.754082679748535, 'logps/chosen': -15.22050952911377, 'logps/rejected': -20.974592208862305, 'accuracy': 0.6000000238418579, 'epoch': 1.07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 1950/5424 [39:15<45:09,  1.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.1611, 'grad_norm': 1.7623517513275146, 'learning_rate': 7.151677593447873e-06, 'rewards/margins': 2.2445762157440186, 'rewards/chosen': -4.35921573638916, 'rewards/rejected': -6.6037917137146, 'logps/margins': 9.957319259643555, 'logps/chosen': -9.668068885803223, 'logps/rejected': -19.62538719177246, 'accuracy': 0.800000011920929, 'epoch': 1.08}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 1960/5424 [39:23<45:57,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.3716, 'grad_norm': 3.946711301803589, 'learning_rate': 7.125499771541786e-06, 'rewards/margins': 0.5358994603157043, 'rewards/chosen': -5.9661865234375, 'rewards/rejected': -6.5020856857299805, 'logps/margins': 1.3615779876708984, 'logps/chosen': -16.92084312438965, 'logps/rejected': -18.282421112060547, 'accuracy': 0.4000000059604645, 'epoch': 1.08}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▋      | 1970/5424 [39:31<45:44,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.3695, 'grad_norm': 3.290703535079956, 'learning_rate': 7.099250641965683e-06, 'rewards/margins': -0.10247936099767685, 'rewards/chosen': -4.65926456451416, 'rewards/rejected': -4.5567851066589355, 'logps/margins': -0.9196411371231079, 'logps/chosen': -10.524124145507812, 'logps/rejected': -9.604482650756836, 'accuracy': 0.4000000059604645, 'epoch': 1.09}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 1980/5424 [39:39<46:59,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.3153, 'grad_norm': 1.7585233449935913, 'learning_rate': 7.072931085342696e-06, 'rewards/margins': 1.4704554080963135, 'rewards/chosen': -7.2597975730896, 'rewards/rejected': -8.730252265930176, 'logps/margins': 11.465753555297852, 'logps/chosen': -18.150598526000977, 'logps/rejected': -29.616348266601562, 'accuracy': 0.800000011920929, 'epoch': 1.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 1990/5424 [39:47<46:24,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.4159, 'grad_norm': 3.316070318222046, 'learning_rate': 7.046541984658693e-06, 'rewards/margins': 2.5944111347198486, 'rewards/chosen': -5.020576000213623, 'rewards/rejected': -7.614987373352051, 'logps/margins': 12.867483139038086, 'logps/chosen': -11.758241653442383, 'logps/rejected': -24.62572479248047, 'accuracy': 0.6000000238418579, 'epoch': 1.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 2000/5424 [39:55<48:00,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.8553, 'grad_norm': 1.861464262008667, 'learning_rate': 7.020084225232648e-06, 'rewards/margins': 1.9599395990371704, 'rewards/chosen': -6.108399868011475, 'rewards/rejected': -8.068340301513672, 'logps/margins': 9.836223602294922, 'logps/chosen': -16.02978515625, 'logps/rejected': -25.866008758544922, 'accuracy': 0.800000011920929, 'epoch': 1.11}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 2010/5424 [40:03<45:28,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.9168, 'grad_norm': 4.4498491287231445, 'learning_rate': 6.993558694686953e-06, 'rewards/margins': 0.8067717552185059, 'rewards/chosen': -4.498985767364502, 'rewards/rejected': -5.30575704574585, 'logps/margins': 3.542581081390381, 'logps/chosen': -10.981904029846191, 'logps/rejected': -14.52448558807373, 'accuracy': 0.699999988079071, 'epoch': 1.11}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 2020/5424 [40:11<45:35,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.819, 'grad_norm': 2.848803997039795, 'learning_rate': 6.966966282917621e-06, 'rewards/margins': 1.8284721374511719, 'rewards/chosen': -5.213123798370361, 'rewards/rejected': -7.041596412658691, 'logps/margins': 11.152040481567383, 'logps/chosen': -11.063289642333984, 'logps/rejected': -22.215328216552734, 'accuracy': 0.800000011920929, 'epoch': 1.12}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 2030/5424 [40:20<45:06,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.5658, 'grad_norm': 1.7970880270004272, 'learning_rate': 6.940307882064443e-06, 'rewards/margins': 1.2852323055267334, 'rewards/chosen': -4.595034122467041, 'rewards/rejected': -5.8802666664123535, 'logps/margins': 7.3541717529296875, 'logps/chosen': -10.548284530639648, 'logps/rejected': -17.9024600982666, 'accuracy': 0.5, 'epoch': 1.12}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 2040/5424 [40:28<45:19,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.2292, 'grad_norm': 3.275700330734253, 'learning_rate': 6.9135843864810555e-06, 'rewards/margins': 4.861614227294922, 'rewards/chosen': -5.173508644104004, 'rewards/rejected': -10.035122871398926, 'logps/margins': 19.023792266845703, 'logps/chosen': -11.949258804321289, 'logps/rejected': -30.973052978515625, 'accuracy': 0.8999999761581421, 'epoch': 1.13}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 2050/5424 [40:36<45:32,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.6407, 'grad_norm': 1.3359869718551636, 'learning_rate': 6.88679669270494e-06, 'rewards/margins': 2.7467007637023926, 'rewards/chosen': -4.315518856048584, 'rewards/rejected': -7.062219142913818, 'logps/margins': 13.870187759399414, 'logps/chosen': -9.884135246276855, 'logps/rejected': -23.754322052001953, 'accuracy': 0.699999988079071, 'epoch': 1.13}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 2060/5424 [40:44<45:13,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.2411, 'grad_norm': 3.8340280055999756, 'learning_rate': 6.859945699427335e-06, 'rewards/margins': 0.9902893304824829, 'rewards/chosen': -5.654347896575928, 'rewards/rejected': -6.644637107849121, 'logps/margins': 3.686051845550537, 'logps/chosen': -14.854772567749023, 'logps/rejected': -18.54082489013672, 'accuracy': 0.6000000238418579, 'epoch': 1.14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 2070/5424 [40:52<44:03,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.7906, 'grad_norm': 2.5901882648468018, 'learning_rate': 6.833032307463097e-06, 'rewards/margins': 3.7321648597717285, 'rewards/chosen': -4.284137725830078, 'rewards/rejected': -8.016302108764648, 'logps/margins': 14.058939933776855, 'logps/chosen': -9.45523452758789, 'logps/rejected': -23.514175415039062, 'accuracy': 0.8999999761581421, 'epoch': 1.14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 2080/5424 [41:00<44:26,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.1021, 'grad_norm': 2.734785318374634, 'learning_rate': 6.806057419720471e-06, 'rewards/margins': 0.7930572032928467, 'rewards/chosen': -5.603710651397705, 'rewards/rejected': -6.396768093109131, 'logps/margins': 4.156748294830322, 'logps/chosen': -14.181230545043945, 'logps/rejected': -18.33797836303711, 'accuracy': 0.6000000238418579, 'epoch': 1.15}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▊      | 2090/5424 [41:09<44:57,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.2769, 'grad_norm': 7.536473751068115, 'learning_rate': 6.779021941170807e-06, 'rewards/margins': 1.35565984249115, 'rewards/chosen': -6.551706790924072, 'rewards/rejected': -7.907366752624512, 'logps/margins': 8.458512306213379, 'logps/chosen': -16.05032730102539, 'logps/rejected': -24.508838653564453, 'accuracy': 0.5, 'epoch': 1.16}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▊      | 2100/5424 [41:17<44:14,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.4058, 'grad_norm': 2.9138598442077637, 'learning_rate': 6.751926778818193e-06, 'rewards/margins': -0.6494985818862915, 'rewards/chosen': -6.4837799072265625, 'rewards/rejected': -5.834280967712402, 'logps/margins': 0.150691419839859, 'logps/chosen': -16.34674072265625, 'logps/rejected': -16.497432708740234, 'accuracy': 0.5, 'epoch': 1.16}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 2110/5424 [41:25<44:11,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 24.2462, 'grad_norm': 7.534541130065918, 'learning_rate': 6.72477284166903e-06, 'rewards/margins': -1.0120036602020264, 'rewards/chosen': -7.211524963378906, 'rewards/rejected': -6.199521064758301, 'logps/margins': -4.764464855194092, 'logps/chosen': -21.64656639099121, 'logps/rejected': -16.88210105895996, 'accuracy': 0.4000000059604645, 'epoch': 1.17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 2120/5424 [41:33<44:07,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.2515, 'grad_norm': 3.199070692062378, 'learning_rate': 6.697561040701535e-06, 'rewards/margins': 2.1049652099609375, 'rewards/chosen': -5.325382232666016, 'rewards/rejected': -7.430347442626953, 'logps/margins': 10.084981918334961, 'logps/chosen': -12.01661205291748, 'logps/rejected': -22.101594924926758, 'accuracy': 0.6000000238418579, 'epoch': 1.17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 2130/5424 [41:41<45:43,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 23.5575, 'grad_norm': 1.8508961200714111, 'learning_rate': 6.670292288835174e-06, 'rewards/margins': 0.036583803594112396, 'rewards/chosen': -7.41208028793335, 'rewards/rejected': -7.448664665222168, 'logps/margins': 3.6363799571990967, 'logps/chosen': -20.980350494384766, 'logps/rejected': -24.616729736328125, 'accuracy': 0.5, 'epoch': 1.18}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 2140/5424 [41:49<45:11,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.1161, 'grad_norm': 3.9220709800720215, 'learning_rate': 6.6429675009000445e-06, 'rewards/margins': 0.708530604839325, 'rewards/chosen': -4.630018711090088, 'rewards/rejected': -5.338549613952637, 'logps/margins': 4.927332878112793, 'logps/chosen': -11.04385757446289, 'logps/rejected': -15.971189498901367, 'accuracy': 0.6000000238418579, 'epoch': 1.18}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███▉      | 2150/5424 [41:57<44:31,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.5104, 'grad_norm': 7.703598499298096, 'learning_rate': 6.615587593606178e-06, 'rewards/margins': 2.158982753753662, 'rewards/chosen': -5.171210289001465, 'rewards/rejected': -7.330192565917969, 'logps/margins': 10.886648178100586, 'logps/chosen': -13.413640975952148, 'logps/rejected': -24.300289154052734, 'accuracy': 0.5, 'epoch': 1.19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███▉      | 2160/5424 [42:06<45:13,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.8083, 'grad_norm': 1.9661282300949097, 'learning_rate': 6.588153485512787e-06, 'rewards/margins': 1.5991209745407104, 'rewards/chosen': -5.09964656829834, 'rewards/rejected': -6.698767185211182, 'logps/margins': 8.304667472839355, 'logps/chosen': -13.173074722290039, 'logps/rejected': -21.477741241455078, 'accuracy': 0.6000000238418579, 'epoch': 1.19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 2170/5424 [42:14<43:38,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 7.1405, 'grad_norm': 2.0346124172210693, 'learning_rate': 6.560666096997447e-06, 'rewards/margins': 1.2721607685089111, 'rewards/chosen': -3.953122615814209, 'rewards/rejected': -5.225283622741699, 'logps/margins': 8.03746223449707, 'logps/chosen': -6.514563083648682, 'logps/rejected': -14.552024841308594, 'accuracy': 0.5, 'epoch': 1.2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 2180/5424 [42:22<44:02,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.4413, 'grad_norm': 4.769764423370361, 'learning_rate': 6.533126350225219e-06, 'rewards/margins': 1.4642242193222046, 'rewards/chosen': -5.236931800842285, 'rewards/rejected': -6.701155185699463, 'logps/margins': 9.554261207580566, 'logps/chosen': -12.795623779296875, 'logps/rejected': -22.349884033203125, 'accuracy': 0.5, 'epoch': 1.21}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 2190/5424 [42:30<43:45,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.5837, 'grad_norm': 1.8198851346969604, 'learning_rate': 6.505535169117716e-06, 'rewards/margins': 1.2823069095611572, 'rewards/chosen': -4.869630813598633, 'rewards/rejected': -6.151937961578369, 'logps/margins': 7.182528495788574, 'logps/chosen': -11.135458946228027, 'logps/rejected': -18.317989349365234, 'accuracy': 0.699999988079071, 'epoch': 1.21}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 2200/5424 [42:38<43:24,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.4898, 'grad_norm': 7.563863754272461, 'learning_rate': 6.477893479322103e-06, 'rewards/margins': 0.9648208618164062, 'rewards/chosen': -6.1728057861328125, 'rewards/rejected': -7.137625694274902, 'logps/margins': 3.4744534492492676, 'logps/chosen': -18.60617446899414, 'logps/rejected': -22.080625534057617, 'accuracy': 0.6000000238418579, 'epoch': 1.22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 2210/5424 [42:46<42:26,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 23.9946, 'grad_norm': 8.419502258300781, 'learning_rate': 6.450202208180043e-06, 'rewards/margins': -1.7731386423110962, 'rewards/chosen': -6.7111358642578125, 'rewards/rejected': -4.937997341156006, 'logps/margins': -8.236177444458008, 'logps/chosen': -20.768512725830078, 'logps/rejected': -12.532332420349121, 'accuracy': 0.5, 'epoch': 1.22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 2220/5424 [42:55<46:26,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.3858, 'grad_norm': 3.7930567264556885, 'learning_rate': 6.422462284696589e-06, 'rewards/margins': 0.5471946597099304, 'rewards/chosen': -6.450594902038574, 'rewards/rejected': -6.9977898597717285, 'logps/margins': 7.1864824295043945, 'logps/chosen': -15.134162902832031, 'logps/rejected': -22.32064437866211, 'accuracy': 0.6000000238418579, 'epoch': 1.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 2230/5424 [43:03<43:17,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.1245, 'grad_norm': 1.3532346487045288, 'learning_rate': 6.394674639509013e-06, 'rewards/margins': 5.074512958526611, 'rewards/chosen': -5.067775249481201, 'rewards/rejected': -10.142288208007812, 'logps/margins': 24.14962387084961, 'logps/chosen': -10.964433670043945, 'logps/rejected': -35.11405563354492, 'accuracy': 0.8999999761581421, 'epoch': 1.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████▏     | 2240/5424 [43:11<42:53,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.7437, 'grad_norm': 6.682137966156006, 'learning_rate': 6.3668402048555875e-06, 'rewards/margins': -0.4514862895011902, 'rewards/chosen': -6.445540428161621, 'rewards/rejected': -5.994054317474365, 'logps/margins': 0.2902478277683258, 'logps/chosen': -17.948856353759766, 'logps/rejected': -18.239105224609375, 'accuracy': 0.5, 'epoch': 1.24}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████▏     | 2250/5424 [43:19<44:04,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.3031, 'grad_norm': 2.2082679271698, 'learning_rate': 6.34174998043458e-06, 'rewards/margins': 1.2996485233306885, 'rewards/chosen': -5.391333103179932, 'rewards/rejected': -6.690981864929199, 'logps/margins': 8.412631034851074, 'logps/chosen': -13.388799667358398, 'logps/rejected': -21.80143165588379, 'accuracy': 0.6000000238418579, 'epoch': 1.24}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 2260/5424 [43:27<43:45,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.5025, 'grad_norm': 5.183680057525635, 'learning_rate': 6.313829219709198e-06, 'rewards/margins': 1.229954481124878, 'rewards/chosen': -5.688626766204834, 'rewards/rejected': -6.918581485748291, 'logps/margins': 6.386670112609863, 'logps/chosen': -14.436622619628906, 'logps/rejected': -20.823291778564453, 'accuracy': 0.5, 'epoch': 1.25}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 2270/5424 [43:36<43:02,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.7766, 'grad_norm': 1.7313724756240845, 'learning_rate': 6.2858643817735e-06, 'rewards/margins': 3.330913543701172, 'rewards/chosen': -4.7090911865234375, 'rewards/rejected': -8.04000473022461, 'logps/margins': 14.765924453735352, 'logps/chosen': -9.152872085571289, 'logps/rejected': -23.91879653930664, 'accuracy': 0.6000000238418579, 'epoch': 1.26}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 2280/5424 [43:44<44:47,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.6546, 'grad_norm': 4.918096542358398, 'learning_rate': 6.257856404810337e-06, 'rewards/margins': 4.754140853881836, 'rewards/chosen': -4.93212890625, 'rewards/rejected': -9.686269760131836, 'logps/margins': 20.492324829101562, 'logps/chosen': -12.288214683532715, 'logps/rejected': -32.780540466308594, 'accuracy': 0.8999999761581421, 'epoch': 1.26}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 2290/5424 [43:52<40:37,  1.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.6315, 'grad_norm': 3.4518120288848877, 'learning_rate': 6.229806228449815e-06, 'rewards/margins': -1.2338947057724, 'rewards/chosen': -5.789928436279297, 'rewards/rejected': -4.556033134460449, 'logps/margins': -6.351327419281006, 'logps/chosen': -17.23817253112793, 'logps/rejected': -10.886845588684082, 'accuracy': 0.4000000059604645, 'epoch': 1.27}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 2300/5424 [44:00<41:13,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.3111, 'grad_norm': 2.601952075958252, 'learning_rate': 6.201714793737775e-06, 'rewards/margins': 2.618009567260742, 'rewards/chosen': -4.453909873962402, 'rewards/rejected': -7.071918487548828, 'logps/margins': 12.344335556030273, 'logps/chosen': -9.887102127075195, 'logps/rejected': -22.23143768310547, 'accuracy': 0.800000011920929, 'epoch': 1.27}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 2310/5424 [44:08<41:35,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.486, 'grad_norm': 2.203310012817383, 'learning_rate': 6.173583043104223e-06, 'rewards/margins': 1.4653985500335693, 'rewards/chosen': -5.039261341094971, 'rewards/rejected': -6.504660129547119, 'logps/margins': 7.341043949127197, 'logps/chosen': -13.456304550170898, 'logps/rejected': -20.79734992980957, 'accuracy': 0.699999988079071, 'epoch': 1.28}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 2320/5424 [44:16<43:30,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.4709, 'grad_norm': 2.0189461708068848, 'learning_rate': 6.145411920331706e-06, 'rewards/margins': 1.3196882009506226, 'rewards/chosen': -5.43789005279541, 'rewards/rejected': -6.757578372955322, 'logps/margins': 8.552888870239258, 'logps/chosen': -11.383638381958008, 'logps/rejected': -19.936527252197266, 'accuracy': 0.6000000238418579, 'epoch': 1.28}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 2330/5424 [44:24<42:10,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.6733, 'grad_norm': 6.138370990753174, 'learning_rate': 6.117202370523656e-06, 'rewards/margins': 1.429430365562439, 'rewards/chosen': -6.307109832763672, 'rewards/rejected': -7.736540794372559, 'logps/margins': 7.130436897277832, 'logps/chosen': -15.539773941040039, 'logps/rejected': -22.670211791992188, 'accuracy': 0.699999988079071, 'epoch': 1.29}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 2340/5424 [44:33<43:27,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.0989, 'grad_norm': 7.097612380981445, 'learning_rate': 6.088955340072675e-06, 'rewards/margins': -0.02454552613198757, 'rewards/chosen': -4.833265781402588, 'rewards/rejected': -4.808720588684082, 'logps/margins': 1.164772391319275, 'logps/chosen': -12.626504898071289, 'logps/rejected': -13.791276931762695, 'accuracy': 0.5, 'epoch': 1.29}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 2350/5424 [44:41<41:14,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 24.3558, 'grad_norm': 2.718549966812134, 'learning_rate': 6.060671776628801e-06, 'rewards/margins': -0.6174829006195068, 'rewards/chosen': -7.202963829040527, 'rewards/rejected': -6.5854811668396, 'logps/margins': -1.917661428451538, 'logps/chosen': -21.83939552307129, 'logps/rejected': -19.921733856201172, 'accuracy': 0.6000000238418579, 'epoch': 1.3}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▎     | 2360/5424 [44:49<40:37,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.5779, 'grad_norm': 5.147314548492432, 'learning_rate': 6.032352629067696e-06, 'rewards/margins': 0.9618139266967773, 'rewards/chosen': -4.7743096351623535, 'rewards/rejected': -5.736123561859131, 'logps/margins': 3.992926836013794, 'logps/chosen': -11.828859329223633, 'logps/rejected': -15.821786880493164, 'accuracy': 0.6000000238418579, 'epoch': 1.31}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▎     | 2370/5424 [44:57<40:37,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.4523, 'grad_norm': 4.83107328414917, 'learning_rate': 6.003998847458825e-06, 'rewards/margins': 0.578163743019104, 'rewards/chosen': -6.572125434875488, 'rewards/rejected': -7.150290012359619, 'logps/margins': 1.5460176467895508, 'logps/chosen': -18.53565216064453, 'logps/rejected': -20.081668853759766, 'accuracy': 0.30000001192092896, 'epoch': 1.31}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 2380/5424 [45:05<40:41,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.656, 'grad_norm': 2.4452438354492188, 'learning_rate': 5.975611383033577e-06, 'rewards/margins': 1.6667474508285522, 'rewards/chosen': -4.99440860748291, 'rewards/rejected': -6.661154747009277, 'logps/margins': 8.471574783325195, 'logps/chosen': -11.953943252563477, 'logps/rejected': -20.42551612854004, 'accuracy': 0.800000011920929, 'epoch': 1.32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 2390/5424 [45:13<40:16,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.5416, 'grad_norm': 6.244542121887207, 'learning_rate': 5.947191188153357e-06, 'rewards/margins': 0.9328299760818481, 'rewards/chosen': -4.803645133972168, 'rewards/rejected': -5.736475467681885, 'logps/margins': 4.419030666351318, 'logps/chosen': -11.412328720092773, 'logps/rejected': -15.831357955932617, 'accuracy': 0.6000000238418579, 'epoch': 1.32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 2400/5424 [45:21<40:14,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.0547, 'grad_norm': 3.9275848865509033, 'learning_rate': 5.91873921627763e-06, 'rewards/margins': -0.48524293303489685, 'rewards/chosen': -5.744604587554932, 'rewards/rejected': -5.25936222076416, 'logps/margins': -3.3590073585510254, 'logps/chosen': -16.209943771362305, 'logps/rejected': -12.850934028625488, 'accuracy': 0.5, 'epoch': 1.33}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 2410/5424 [45:29<40:10,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.0235, 'grad_norm': 3.1411426067352295, 'learning_rate': 5.89025642193194e-06, 'rewards/margins': 3.4825596809387207, 'rewards/chosen': -4.153059959411621, 'rewards/rejected': -7.6356201171875, 'logps/margins': 17.1350040435791, 'logps/chosen': -7.7580885887146, 'logps/rejected': -24.89309310913086, 'accuracy': 0.8999999761581421, 'epoch': 1.33}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▍     | 2420/5424 [45:37<39:45,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.2667, 'grad_norm': 2.3392817974090576, 'learning_rate': 5.861743760675882e-06, 'rewards/margins': 0.793597936630249, 'rewards/chosen': -6.110128402709961, 'rewards/rejected': -6.903726100921631, 'logps/margins': 3.7983322143554688, 'logps/chosen': -16.835908889770508, 'logps/rejected': -20.63424301147461, 'accuracy': 0.6000000238418579, 'epoch': 1.34}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▍     | 2430/5424 [45:45<40:07,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.2834, 'grad_norm': 6.571786880493164, 'learning_rate': 5.833202189071045e-06, 'rewards/margins': 0.2476474791765213, 'rewards/chosen': -5.379379749298096, 'rewards/rejected': -5.62702751159668, 'logps/margins': 4.334205627441406, 'logps/chosen': -11.935916900634766, 'logps/rejected': -16.270122528076172, 'accuracy': 0.4000000059604645, 'epoch': 1.34}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▍     | 2440/5424 [45:53<39:19,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.2224, 'grad_norm': 2.5460522174835205, 'learning_rate': 5.804632664648919e-06, 'rewards/margins': 0.8136190176010132, 'rewards/chosen': -4.565447807312012, 'rewards/rejected': -5.3790669441223145, 'logps/margins': 2.7010464668273926, 'logps/chosen': -11.024322509765625, 'logps/rejected': -13.725369453430176, 'accuracy': 0.5, 'epoch': 1.35}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 2450/5424 [46:01<41:48,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.627, 'grad_norm': 2.9094834327697754, 'learning_rate': 5.776036145878781e-06, 'rewards/margins': 1.5328788757324219, 'rewards/chosen': -4.970126152038574, 'rewards/rejected': -6.503005027770996, 'logps/margins': 6.3334760665893555, 'logps/chosen': -11.78812313079834, 'logps/rejected': -18.121599197387695, 'accuracy': 0.6000000238418579, 'epoch': 1.36}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 2460/5424 [46:10<39:40,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.1332, 'grad_norm': 0.5153314471244812, 'learning_rate': 5.747413592135526e-06, 'rewards/margins': 1.5174522399902344, 'rewards/chosen': -6.134283542633057, 'rewards/rejected': -7.651736259460449, 'logps/margins': 7.76747989654541, 'logps/chosen': -15.091796875, 'logps/rejected': -22.85927391052246, 'accuracy': 0.6000000238418579, 'epoch': 1.36}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 2470/5424 [46:18<40:14,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.5374, 'grad_norm': 8.191039085388184, 'learning_rate': 5.7187659636674896e-06, 'rewards/margins': 2.858473777770996, 'rewards/chosen': -5.306985855102539, 'rewards/rejected': -8.165457725524902, 'logps/margins': 13.345113754272461, 'logps/chosen': -13.815221786499023, 'logps/rejected': -27.16033935546875, 'accuracy': 0.800000011920929, 'epoch': 1.37}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 2480/5424 [46:26<39:09,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.1379, 'grad_norm': 2.022143840789795, 'learning_rate': 5.690094221564231e-06, 'rewards/margins': 2.2296109199523926, 'rewards/chosen': -4.0900373458862305, 'rewards/rejected': -6.319647789001465, 'logps/margins': 10.860811233520508, 'logps/chosen': -8.721738815307617, 'logps/rejected': -19.582550048828125, 'accuracy': 0.800000011920929, 'epoch': 1.37}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 2490/5424 [46:34<38:47,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.321, 'grad_norm': 2.412459373474121, 'learning_rate': 5.661399327724287e-06, 'rewards/margins': 1.6493899822235107, 'rewards/chosen': -4.560320854187012, 'rewards/rejected': -6.20971155166626, 'logps/margins': 7.044615745544434, 'logps/chosen': -9.686103820800781, 'logps/rejected': -16.7307186126709, 'accuracy': 0.699999988079071, 'epoch': 1.38}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 2500/5424 [46:42<40:13,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.7028, 'grad_norm': 7.455387115478516, 'learning_rate': 5.63268224482291e-06, 'rewards/margins': 1.1170469522476196, 'rewards/chosen': -5.201576232910156, 'rewards/rejected': -6.318623065948486, 'logps/margins': 7.4744391441345215, 'logps/chosen': -11.591773986816406, 'logps/rejected': -19.066213607788086, 'accuracy': 0.699999988079071, 'epoch': 1.38}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▋     | 2510/5424 [46:51<40:52,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.1647, 'grad_norm': 2.649550676345825, 'learning_rate': 5.603943936279759e-06, 'rewards/margins': 3.8750946521759033, 'rewards/chosen': -5.4466681480407715, 'rewards/rejected': -9.321763038635254, 'logps/margins': 18.018014907836914, 'logps/chosen': -13.45813274383545, 'logps/rejected': -31.476146697998047, 'accuracy': 0.699999988079071, 'epoch': 1.39}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▋     | 2520/5424 [46:59<39:39,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.9216, 'grad_norm': 3.5919888019561768, 'learning_rate': 5.575185366226589e-06, 'rewards/margins': 1.6171715259552002, 'rewards/chosen': -5.825490474700928, 'rewards/rejected': -7.442663669586182, 'logps/margins': 7.670951843261719, 'logps/chosen': -15.941302299499512, 'logps/rejected': -23.612255096435547, 'accuracy': 0.699999988079071, 'epoch': 1.39}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 2530/5424 [47:07<38:20,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 7.8171, 'grad_norm': 3.548208236694336, 'learning_rate': 5.546407499474901e-06, 'rewards/margins': 1.899463415145874, 'rewards/chosen': -4.0772705078125, 'rewards/rejected': -5.976734638214111, 'logps/margins': 10.903752326965332, 'logps/chosen': -7.361275672912598, 'logps/rejected': -18.26502799987793, 'accuracy': 0.699999988079071, 'epoch': 1.4}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 2540/5424 [47:15<40:42,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.9069, 'grad_norm': 2.897167921066284, 'learning_rate': 5.517611301483571e-06, 'rewards/margins': 2.208590030670166, 'rewards/chosen': -5.9553022384643555, 'rewards/rejected': -8.16389274597168, 'logps/margins': 13.472488403320312, 'logps/chosen': -13.97906494140625, 'logps/rejected': -27.451553344726562, 'accuracy': 0.699999988079071, 'epoch': 1.4}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 2550/5424 [47:24<40:39,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 26.4504, 'grad_norm': 3.2508704662323, 'learning_rate': 5.488797738326468e-06, 'rewards/margins': -1.0593302249908447, 'rewards/chosen': -8.17825984954834, 'rewards/rejected': -7.118929862976074, 'logps/margins': -2.5031137466430664, 'logps/chosen': -23.565650939941406, 'logps/rejected': -21.062538146972656, 'accuracy': 0.4000000059604645, 'epoch': 1.41}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 2560/5424 [47:32<39:58,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.6679, 'grad_norm': 2.6064109802246094, 'learning_rate': 5.459967776660038e-06, 'rewards/margins': 3.0067882537841797, 'rewards/chosen': -5.124293327331543, 'rewards/rejected': -8.131082534790039, 'logps/margins': 14.97874641418457, 'logps/chosen': -11.128256797790527, 'logps/rejected': -26.10700035095215, 'accuracy': 0.800000011920929, 'epoch': 1.42}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 2570/5424 [47:40<39:14,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.7914, 'grad_norm': 7.518320083618164, 'learning_rate': 5.431122383690872e-06, 'rewards/margins': 1.8871015310287476, 'rewards/chosen': -6.348234176635742, 'rewards/rejected': -8.235335350036621, 'logps/margins': 10.004353523254395, 'logps/chosen': -16.551109313964844, 'logps/rejected': -26.555461883544922, 'accuracy': 0.6000000238418579, 'epoch': 1.42}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 2580/5424 [47:49<41:37,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.0063, 'grad_norm': 3.434844970703125, 'learning_rate': 5.402262527143267e-06, 'rewards/margins': 1.3902136087417603, 'rewards/chosen': -6.518390655517578, 'rewards/rejected': -7.908604621887207, 'logps/margins': 6.879942417144775, 'logps/chosen': -18.789775848388672, 'logps/rejected': -25.669719696044922, 'accuracy': 0.699999988079071, 'epoch': 1.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 2590/5424 [47:57<38:11,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.8546, 'grad_norm': 9.570062637329102, 'learning_rate': 5.373389175226745e-06, 'rewards/margins': 1.3831679821014404, 'rewards/chosen': -4.966973304748535, 'rewards/rejected': -6.3501410484313965, 'logps/margins': 6.888694763183594, 'logps/chosen': -11.000883102416992, 'logps/rejected': -17.889577865600586, 'accuracy': 0.800000011920929, 'epoch': 1.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 2600/5424 [48:05<38:26,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.1334, 'grad_norm': 3.826204776763916, 'learning_rate': 5.344503296603588e-06, 'rewards/margins': 3.0969812870025635, 'rewards/chosen': -4.814194679260254, 'rewards/rejected': -7.9111762046813965, 'logps/margins': 15.734753608703613, 'logps/chosen': -9.548188209533691, 'logps/rejected': -25.282939910888672, 'accuracy': 0.699999988079071, 'epoch': 1.44}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 2610/5424 [48:13<39:30,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.3794, 'grad_norm': 4.58826208114624, 'learning_rate': 5.315605860356324e-06, 'rewards/margins': 2.095954179763794, 'rewards/chosen': -4.997206211090088, 'rewards/rejected': -7.093161106109619, 'logps/margins': 10.21397590637207, 'logps/chosen': -11.554400444030762, 'logps/rejected': -21.768375396728516, 'accuracy': 0.699999988079071, 'epoch': 1.44}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 2620/5424 [48:22<41:31,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.8206, 'grad_norm': 1.952534556388855, 'learning_rate': 5.286697835955231e-06, 'rewards/margins': 2.989589214324951, 'rewards/chosen': -5.873326778411865, 'rewards/rejected': -8.862915992736816, 'logps/margins': 12.15410041809082, 'logps/chosen': -14.984341621398926, 'logps/rejected': -27.138442993164062, 'accuracy': 0.8999999761581421, 'epoch': 1.45}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 2630/5424 [48:30<36:41,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.2477, 'grad_norm': 1.8750946521759033, 'learning_rate': 5.257780193225803e-06, 'rewards/margins': 1.4676834344863892, 'rewards/chosen': -5.768973350524902, 'rewards/rejected': -7.236656188964844, 'logps/margins': 5.1747145652771, 'logps/chosen': -16.82256507873535, 'logps/rejected': -21.99728012084961, 'accuracy': 0.6000000238418579, 'epoch': 1.45}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▊     | 2640/5424 [48:38<37:02,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.0096, 'grad_norm': 2.873603582382202, 'learning_rate': 5.228853902316213e-06, 'rewards/margins': 2.377222776412964, 'rewards/chosen': -4.648719310760498, 'rewards/rejected': -7.025942325592041, 'logps/margins': 12.17603874206543, 'logps/chosen': -11.049688339233398, 'logps/rejected': -23.225727081298828, 'accuracy': 0.699999988079071, 'epoch': 1.46}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 2650/5424 [48:46<36:18,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.6002, 'grad_norm': 3.3672549724578857, 'learning_rate': 5.1999199336647745e-06, 'rewards/margins': 0.5245831608772278, 'rewards/chosen': -4.9557600021362305, 'rewards/rejected': -5.480343341827393, 'logps/margins': 0.48953431844711304, 'logps/chosen': -12.334794998168945, 'logps/rejected': -12.824328422546387, 'accuracy': 0.4000000059604645, 'epoch': 1.47}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 2660/5424 [48:54<38:45,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.2489, 'grad_norm': 6.228443145751953, 'learning_rate': 5.170979257967372e-06, 'rewards/margins': 0.1301266849040985, 'rewards/chosen': -6.493707180023193, 'rewards/rejected': -6.623833656311035, 'logps/margins': 1.6724551916122437, 'logps/chosen': -16.486635208129883, 'logps/rejected': -18.15909194946289, 'accuracy': 0.5, 'epoch': 1.47}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 2670/5424 [49:02<37:30,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.3865, 'grad_norm': 3.426976203918457, 'learning_rate': 5.142032846144911e-06, 'rewards/margins': 2.3487422466278076, 'rewards/chosen': -5.22392463684082, 'rewards/rejected': -7.572667598724365, 'logps/margins': 14.345556259155273, 'logps/chosen': -12.23628044128418, 'logps/rejected': -26.581838607788086, 'accuracy': 0.800000011920929, 'epoch': 1.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 2680/5424 [49:10<36:33,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.4358, 'grad_norm': 9.929608345031738, 'learning_rate': 5.113081669310728e-06, 'rewards/margins': 0.6742843985557556, 'rewards/chosen': -5.503249645233154, 'rewards/rejected': -6.1775336265563965, 'logps/margins': 1.7025928497314453, 'logps/chosen': -15.410467147827148, 'logps/rejected': -17.113061904907227, 'accuracy': 0.5, 'epoch': 1.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|████▉     | 2690/5424 [49:18<36:22,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.0823, 'grad_norm': 6.769019603729248, 'learning_rate': 5.0841266987380234e-06, 'rewards/margins': 1.071721076965332, 'rewards/chosen': -5.4988203048706055, 'rewards/rejected': -6.5705413818359375, 'logps/margins': 5.766303062438965, 'logps/chosen': -14.683209419250488, 'logps/rejected': -20.44951057434082, 'accuracy': 0.6000000238418579, 'epoch': 1.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|████▉     | 2700/5424 [49:27<36:55,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.564, 'grad_norm': 1.498840570449829, 'learning_rate': 5.055168905827271e-06, 'rewards/margins': 1.595594048500061, 'rewards/chosen': -5.4366135597229, 'rewards/rejected': -7.0322065353393555, 'logps/margins': 7.159131050109863, 'logps/chosen': -13.241172790527344, 'logps/rejected': -20.400304794311523, 'accuracy': 0.4000000059604645, 'epoch': 1.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|████▉     | 2710/5424 [49:35<38:34,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 21.6796, 'grad_norm': 2.0862083435058594, 'learning_rate': 5.026209262073634e-06, 'rewards/margins': -0.22706374526023865, 'rewards/chosen': -7.1690263748168945, 'rewards/rejected': -6.941962242126465, 'logps/margins': 0.9853447079658508, 'logps/chosen': -19.45855140686035, 'logps/rejected': -20.443897247314453, 'accuracy': 0.5, 'epoch': 1.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 2712/5424 [49:37<37:32,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating response...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/201 [00:00<?, ?it/s]\u001b[A\n",
      "  1%|          | 2/201 [00:00<01:06,  3.01it/s]\u001b[A\n",
      "  1%|▏         | 3/201 [00:01<01:42,  1.93it/s]\u001b[A\n",
      "  2%|▏         | 4/201 [00:02<01:48,  1.82it/s]\u001b[A\n",
      "  2%|▏         | 5/201 [00:02<01:54,  1.71it/s]\u001b[A\n",
      "  3%|▎         | 6/201 [00:03<01:58,  1.64it/s]\u001b[A\n",
      "  3%|▎         | 7/201 [00:04<02:01,  1.60it/s]\u001b[A\n",
      "  4%|▍         | 8/201 [00:04<02:03,  1.56it/s]\u001b[A\n",
      "  4%|▍         | 9/201 [00:05<02:11,  1.46it/s]\u001b[A\n",
      "  5%|▍         | 10/201 [00:06<02:05,  1.52it/s]\u001b[A\n",
      "  5%|▌         | 11/201 [00:06<02:05,  1.52it/s]\u001b[A\n",
      "  6%|▌         | 12/201 [00:07<02:05,  1.51it/s]\u001b[A\n",
      "  6%|▋         | 13/201 [00:08<02:10,  1.44it/s]\u001b[A\n",
      "  7%|▋         | 14/201 [00:08<02:07,  1.47it/s]\u001b[A\n",
      "  7%|▋         | 15/201 [00:09<02:06,  1.47it/s]\u001b[A\n",
      "  8%|▊         | 16/201 [00:10<02:05,  1.48it/s]\u001b[A\n",
      "  8%|▊         | 17/201 [00:10<02:09,  1.42it/s]\u001b[A\n",
      "  9%|▉         | 18/201 [00:11<02:08,  1.42it/s]\u001b[A\n",
      "  9%|▉         | 19/201 [00:12<02:08,  1.42it/s]\u001b[A\n",
      " 10%|▉         | 20/201 [00:13<02:05,  1.44it/s]\u001b[A\n",
      " 10%|█         | 21/201 [00:13<02:08,  1.40it/s]\u001b[A\n",
      " 11%|█         | 22/201 [00:14<02:07,  1.41it/s]\u001b[A\n",
      " 11%|█▏        | 23/201 [00:15<02:11,  1.36it/s]\u001b[A\n",
      " 12%|█▏        | 24/201 [00:15<02:05,  1.41it/s]\u001b[A\n",
      " 12%|█▏        | 25/201 [00:16<02:03,  1.43it/s]\u001b[A\n",
      " 13%|█▎        | 26/201 [00:17<02:00,  1.46it/s]\u001b[A\n",
      " 13%|█▎        | 27/201 [00:17<01:58,  1.47it/s]\u001b[A\n",
      " 14%|█▍        | 28/201 [00:18<02:02,  1.41it/s]\u001b[A\n",
      " 14%|█▍        | 29/201 [00:19<02:00,  1.43it/s]\u001b[A\n",
      " 15%|█▍        | 30/201 [00:20<01:57,  1.45it/s]\u001b[A\n",
      " 15%|█▌        | 31/201 [00:20<01:55,  1.48it/s]\u001b[A\n",
      " 16%|█▌        | 32/201 [00:21<01:52,  1.50it/s]\u001b[A\n",
      " 16%|█▋        | 33/201 [00:22<01:57,  1.43it/s]\u001b[A\n",
      " 17%|█▋        | 34/201 [00:22<01:55,  1.45it/s]\u001b[A\n",
      " 17%|█▋        | 35/201 [00:23<01:52,  1.47it/s]\u001b[A\n",
      " 18%|█▊        | 36/201 [00:24<01:52,  1.47it/s]\u001b[A\n",
      " 18%|█▊        | 37/201 [00:24<01:51,  1.47it/s]\u001b[A\n",
      " 19%|█▉        | 38/201 [00:25<01:49,  1.49it/s]\u001b[A\n",
      " 19%|█▉        | 39/201 [00:26<01:49,  1.48it/s]\u001b[A\n",
      " 20%|█▉        | 40/201 [00:26<01:47,  1.49it/s]\u001b[A\n",
      " 20%|██        | 41/201 [00:27<01:43,  1.54it/s]\u001b[A\n",
      " 21%|██        | 42/201 [00:28<01:49,  1.45it/s]\u001b[A\n",
      " 21%|██▏       | 43/201 [00:28<01:46,  1.48it/s]\u001b[A\n",
      " 22%|██▏       | 44/201 [00:29<01:45,  1.49it/s]\u001b[A\n",
      " 22%|██▏       | 45/201 [00:30<01:48,  1.43it/s]\u001b[A\n",
      " 23%|██▎       | 46/201 [00:30<01:46,  1.45it/s]\u001b[A\n",
      " 23%|██▎       | 47/201 [00:31<01:45,  1.46it/s]\u001b[A\n",
      " 24%|██▍       | 48/201 [00:32<01:40,  1.52it/s]\u001b[A\n",
      " 24%|██▍       | 49/201 [00:32<01:37,  1.56it/s]\u001b[A\n",
      " 25%|██▍       | 50/201 [00:33<01:37,  1.54it/s]\u001b[A\n",
      " 25%|██▌       | 51/201 [00:34<01:37,  1.54it/s]\u001b[A\n",
      " 26%|██▌       | 52/201 [00:34<01:37,  1.53it/s]\u001b[A\n",
      " 26%|██▋       | 53/201 [00:35<01:37,  1.52it/s]\u001b[A\n",
      " 27%|██▋       | 54/201 [00:36<01:41,  1.45it/s]\u001b[A\n",
      " 27%|██▋       | 55/201 [00:36<01:39,  1.47it/s]\u001b[A\n",
      " 28%|██▊       | 56/201 [00:37<01:37,  1.48it/s]\u001b[A\n",
      " 28%|██▊       | 57/201 [00:38<01:41,  1.41it/s]\u001b[A\n",
      " 29%|██▉       | 58/201 [00:38<01:39,  1.43it/s]\u001b[A\n",
      " 29%|██▉       | 59/201 [00:39<01:37,  1.46it/s]\u001b[A\n",
      " 30%|██▉       | 60/201 [00:40<01:35,  1.48it/s]\u001b[A\n",
      " 30%|███       | 61/201 [00:40<01:34,  1.49it/s]\u001b[A\n",
      " 31%|███       | 62/201 [00:41<01:33,  1.49it/s]\u001b[A\n",
      " 31%|███▏      | 63/201 [00:42<01:33,  1.48it/s]\u001b[A\n",
      " 32%|███▏      | 64/201 [00:42<01:32,  1.49it/s]\u001b[A\n",
      " 32%|███▏      | 65/201 [00:43<01:31,  1.48it/s]\u001b[A\n",
      " 33%|███▎      | 66/201 [00:44<01:35,  1.42it/s]\u001b[A\n",
      " 33%|███▎      | 67/201 [00:45<01:33,  1.43it/s]\u001b[A\n",
      " 34%|███▍      | 68/201 [00:45<01:31,  1.45it/s]\u001b[A\n",
      " 34%|███▍      | 69/201 [00:46<01:27,  1.51it/s]\u001b[A\n",
      " 35%|███▍      | 70/201 [00:47<01:26,  1.51it/s]\u001b[A\n",
      " 35%|███▌      | 71/201 [00:47<01:26,  1.50it/s]\u001b[A\n",
      " 36%|███▌      | 72/201 [00:48<01:25,  1.51it/s]\u001b[A\n",
      " 36%|███▋      | 73/201 [00:49<01:25,  1.50it/s]\u001b[A\n",
      " 37%|███▋      | 74/201 [00:49<01:23,  1.51it/s]\u001b[A\n",
      " 37%|███▋      | 75/201 [00:50<01:27,  1.43it/s]\u001b[A\n",
      " 38%|███▊      | 76/201 [00:51<01:26,  1.45it/s]\u001b[A\n",
      " 38%|███▊      | 77/201 [00:51<01:24,  1.46it/s]\u001b[A\n",
      " 39%|███▉      | 78/201 [00:52<01:27,  1.41it/s]\u001b[A\n",
      " 39%|███▉      | 79/201 [00:53<01:29,  1.37it/s]\u001b[A\n",
      " 40%|███▉      | 80/201 [00:54<01:29,  1.34it/s]\u001b[A\n",
      " 40%|████      | 81/201 [00:54<01:25,  1.40it/s]\u001b[A\n",
      " 41%|████      | 82/201 [00:55<01:28,  1.35it/s]\u001b[A\n",
      " 41%|████▏     | 83/201 [00:56<01:28,  1.33it/s]\u001b[A\n",
      " 42%|████▏     | 84/201 [00:57<01:24,  1.39it/s]\u001b[A\n",
      " 42%|████▏     | 85/201 [00:57<01:21,  1.43it/s]\u001b[A\n",
      " 43%|████▎     | 86/201 [00:58<01:18,  1.46it/s]\u001b[A\n",
      " 43%|████▎     | 87/201 [00:58<01:16,  1.49it/s]\u001b[A\n",
      " 44%|████▍     | 88/201 [00:59<01:15,  1.50it/s]\u001b[A\n",
      " 44%|████▍     | 89/201 [01:00<01:18,  1.42it/s]\u001b[A\n",
      " 45%|████▍     | 90/201 [01:01<01:16,  1.46it/s]\u001b[A\n",
      " 45%|████▌     | 91/201 [01:01<01:12,  1.51it/s]\u001b[A\n",
      " 46%|████▌     | 92/201 [01:02<01:15,  1.44it/s]\u001b[A\n",
      " 46%|████▋     | 93/201 [01:03<01:14,  1.46it/s]\u001b[A\n",
      " 47%|████▋     | 94/201 [01:03<01:12,  1.47it/s]\u001b[A\n",
      " 47%|████▋     | 95/201 [01:04<01:11,  1.48it/s]\u001b[A\n",
      " 48%|████▊     | 96/201 [01:05<01:08,  1.53it/s]\u001b[A\n",
      " 48%|████▊     | 97/201 [01:05<01:08,  1.53it/s]\u001b[A\n",
      " 49%|████▉     | 98/201 [01:06<01:07,  1.53it/s]\u001b[A\n",
      " 49%|████▉     | 99/201 [01:07<01:07,  1.52it/s]\u001b[A\n",
      " 50%|████▉     | 100/201 [01:07<01:06,  1.52it/s]\u001b[A\n",
      " 50%|█████     | 101/201 [01:08<01:05,  1.52it/s]\u001b[A\n",
      " 51%|█████     | 102/201 [01:08<01:04,  1.53it/s]\u001b[A\n",
      " 51%|█████     | 103/201 [01:09<01:04,  1.51it/s]\u001b[A\n",
      " 52%|█████▏    | 104/201 [01:10<01:05,  1.49it/s]\u001b[A\n",
      " 52%|█████▏    | 105/201 [01:11<01:04,  1.48it/s]\u001b[A\n",
      " 53%|█████▎    | 106/201 [01:11<01:03,  1.50it/s]\u001b[A\n",
      " 53%|█████▎    | 107/201 [01:12<01:02,  1.51it/s]\u001b[A\n",
      " 54%|█████▎    | 108/201 [01:12<01:01,  1.52it/s]\u001b[A\n",
      " 54%|█████▍    | 109/201 [01:13<01:00,  1.53it/s]\u001b[A\n",
      " 55%|█████▍    | 110/201 [01:14<00:58,  1.56it/s]\u001b[A\n",
      " 55%|█████▌    | 111/201 [01:14<00:58,  1.54it/s]\u001b[A\n",
      " 56%|█████▌    | 112/201 [01:15<00:58,  1.52it/s]\u001b[A\n",
      " 56%|█████▌    | 113/201 [01:16<00:57,  1.53it/s]\u001b[A\n",
      " 57%|█████▋    | 114/201 [01:16<00:56,  1.53it/s]\u001b[A\n",
      " 57%|█████▋    | 115/201 [01:17<00:57,  1.50it/s]\u001b[A\n",
      " 58%|█████▊    | 116/201 [01:18<00:56,  1.52it/s]\u001b[A\n",
      " 58%|█████▊    | 117/201 [01:18<00:55,  1.52it/s]\u001b[A\n",
      " 59%|█████▊    | 118/201 [01:19<00:54,  1.53it/s]\u001b[A\n",
      " 59%|█████▉    | 119/201 [01:20<00:53,  1.53it/s]\u001b[A\n",
      " 60%|█████▉    | 120/201 [01:20<00:53,  1.53it/s]\u001b[A\n",
      " 60%|██████    | 121/201 [01:21<00:52,  1.53it/s]\u001b[A\n",
      " 61%|██████    | 122/201 [01:22<00:52,  1.51it/s]\u001b[A\n",
      " 61%|██████    | 123/201 [01:22<00:53,  1.45it/s]\u001b[A\n",
      " 62%|██████▏   | 124/201 [01:23<00:52,  1.46it/s]\u001b[A\n",
      " 62%|██████▏   | 125/201 [01:24<00:50,  1.52it/s]\u001b[A\n",
      " 63%|██████▎   | 126/201 [01:24<00:48,  1.56it/s]\u001b[A\n",
      " 63%|██████▎   | 127/201 [01:25<00:47,  1.55it/s]\u001b[A\n",
      " 64%|██████▎   | 128/201 [01:26<00:47,  1.54it/s]\u001b[A\n",
      " 64%|██████▍   | 129/201 [01:26<00:46,  1.54it/s]\u001b[A\n",
      " 65%|██████▍   | 130/201 [01:27<00:46,  1.54it/s]\u001b[A\n",
      " 65%|██████▌   | 131/201 [01:28<00:45,  1.54it/s]\u001b[A\n",
      " 66%|██████▌   | 132/201 [01:28<00:49,  1.40it/s]\u001b[A\n",
      " 66%|██████▌   | 133/201 [01:29<00:48,  1.41it/s]\u001b[A\n",
      " 67%|██████▋   | 134/201 [01:30<00:46,  1.44it/s]\u001b[A\n",
      " 67%|██████▋   | 135/201 [01:30<00:44,  1.47it/s]\u001b[A\n",
      " 68%|██████▊   | 136/201 [01:31<00:43,  1.49it/s]\u001b[A\n",
      " 68%|██████▊   | 137/201 [01:32<00:43,  1.49it/s]\u001b[A\n",
      " 69%|██████▊   | 138/201 [01:32<00:42,  1.49it/s]\u001b[A\n",
      " 69%|██████▉   | 139/201 [01:33<00:41,  1.51it/s]\u001b[A\n",
      " 70%|██████▉   | 140/201 [01:34<00:40,  1.51it/s]\u001b[A\n",
      " 70%|███████   | 141/201 [01:34<00:39,  1.51it/s]\u001b[A\n",
      " 71%|███████   | 142/201 [01:35<00:41,  1.44it/s]\u001b[A\n",
      " 71%|███████   | 143/201 [01:36<00:39,  1.45it/s]\u001b[A\n",
      " 72%|███████▏  | 144/201 [01:36<00:39,  1.46it/s]\u001b[A\n",
      " 72%|███████▏  | 145/201 [01:37<00:37,  1.49it/s]\u001b[A\n",
      " 73%|███████▎  | 146/201 [01:38<00:38,  1.42it/s]\u001b[A\n",
      " 73%|███████▎  | 147/201 [01:39<00:39,  1.37it/s]\u001b[A\n",
      " 74%|███████▎  | 148/201 [01:39<00:37,  1.42it/s]\u001b[A\n",
      " 74%|███████▍  | 149/201 [01:40<00:36,  1.44it/s]\u001b[A\n",
      " 75%|███████▍  | 150/201 [01:41<00:35,  1.46it/s]\u001b[A\n",
      " 75%|███████▌  | 151/201 [01:41<00:35,  1.39it/s]\u001b[A\n",
      " 76%|███████▌  | 152/201 [01:42<00:34,  1.42it/s]\u001b[A\n",
      " 76%|███████▌  | 153/201 [01:43<00:33,  1.44it/s]\u001b[A\n",
      " 77%|███████▋  | 154/201 [01:43<00:31,  1.47it/s]\u001b[A\n",
      " 77%|███████▋  | 155/201 [01:44<00:31,  1.47it/s]\u001b[A\n",
      " 78%|███████▊  | 156/201 [01:45<00:30,  1.49it/s]\u001b[A\n",
      " 78%|███████▊  | 157/201 [01:45<00:29,  1.47it/s]\u001b[A\n",
      " 79%|███████▊  | 158/201 [01:46<00:28,  1.49it/s]\u001b[A\n",
      " 79%|███████▉  | 159/201 [01:47<00:27,  1.51it/s]\u001b[A\n",
      " 80%|███████▉  | 160/201 [01:47<00:26,  1.52it/s]\u001b[A\n",
      " 80%|████████  | 161/201 [01:48<00:26,  1.53it/s]\u001b[A\n",
      " 81%|████████  | 162/201 [01:49<00:25,  1.53it/s]\u001b[A\n",
      " 81%|████████  | 163/201 [01:49<00:24,  1.52it/s]\u001b[A\n",
      " 82%|████████▏ | 164/201 [01:50<00:24,  1.52it/s]\u001b[A\n",
      " 82%|████████▏ | 165/201 [01:51<00:24,  1.49it/s]\u001b[A\n",
      " 83%|████████▎ | 166/201 [01:51<00:23,  1.49it/s]\u001b[A\n",
      " 83%|████████▎ | 167/201 [01:52<00:22,  1.49it/s]\u001b[A\n",
      " 84%|████████▎ | 168/201 [01:53<00:23,  1.40it/s]\u001b[A\n",
      " 84%|████████▍ | 169/201 [01:54<00:22,  1.43it/s]\u001b[A\n",
      " 85%|████████▍ | 170/201 [01:54<00:21,  1.44it/s]\u001b[A\n",
      " 85%|████████▌ | 171/201 [01:55<00:20,  1.47it/s]\u001b[A\n",
      " 86%|████████▌ | 172/201 [01:56<00:20,  1.41it/s]\u001b[A\n",
      " 86%|████████▌ | 173/201 [01:56<00:19,  1.43it/s]\u001b[A\n",
      " 87%|████████▋ | 174/201 [01:57<00:18,  1.45it/s]\u001b[A\n",
      " 87%|████████▋ | 175/201 [01:58<00:17,  1.46it/s]\u001b[A\n",
      " 88%|████████▊ | 176/201 [01:58<00:16,  1.48it/s]\u001b[A\n",
      " 88%|████████▊ | 177/201 [01:59<00:16,  1.49it/s]\u001b[A\n",
      " 89%|████████▊ | 178/201 [02:00<00:15,  1.49it/s]\u001b[A\n",
      " 89%|████████▉ | 179/201 [02:00<00:14,  1.51it/s]\u001b[A\n",
      " 90%|████████▉ | 180/201 [02:01<00:13,  1.52it/s]\u001b[A\n",
      " 90%|█████████ | 181/201 [02:02<00:13,  1.52it/s]\u001b[A\n",
      " 91%|█████████ | 182/201 [02:02<00:12,  1.52it/s]\u001b[A\n",
      " 91%|█████████ | 183/201 [02:03<00:11,  1.53it/s]\u001b[A\n",
      " 92%|█████████▏| 184/201 [02:04<00:11,  1.52it/s]\u001b[A\n",
      " 92%|█████████▏| 185/201 [02:04<00:10,  1.52it/s]\u001b[A\n",
      " 93%|█████████▎| 186/201 [02:05<00:09,  1.51it/s]\u001b[A\n",
      " 93%|█████████▎| 187/201 [02:06<00:09,  1.50it/s]\u001b[A\n",
      " 94%|█████████▎| 188/201 [02:06<00:08,  1.50it/s]\u001b[A\n",
      " 94%|█████████▍| 189/201 [02:07<00:07,  1.50it/s]\u001b[A\n",
      " 95%|█████████▍| 190/201 [02:08<00:07,  1.50it/s]\u001b[A\n",
      " 95%|█████████▌| 191/201 [02:08<00:06,  1.51it/s]\u001b[A\n",
      " 96%|█████████▌| 192/201 [02:09<00:05,  1.52it/s]\u001b[A\n",
      " 96%|█████████▌| 193/201 [02:10<00:05,  1.51it/s]\u001b[A\n",
      " 97%|█████████▋| 194/201 [02:10<00:04,  1.55it/s]\u001b[A\n",
      " 97%|█████████▋| 195/201 [02:11<00:03,  1.55it/s]\u001b[A\n",
      " 98%|█████████▊| 196/201 [02:12<00:03,  1.53it/s]\u001b[A\n",
      " 98%|█████████▊| 197/201 [02:12<00:02,  1.52it/s]\u001b[A\n",
      " 99%|█████████▊| 198/201 [02:13<00:01,  1.53it/s]\u001b[A\n",
      " 99%|█████████▉| 199/201 [02:13<00:01,  1.53it/s]\u001b[A\n",
      "100%|█████████▉| 200/201 [02:14<00:00,  1.45it/s]\u001b[A\n",
      "                                                   A\n",
      " 50%|█████     | 2712/5424 [51:58<37:32,  1.20it/s]\n",
      "100%|██████████| 201/201 [02:15<00:00,  1.48it/s]\u001b[A\n",
      "                                                 \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 14.656996726989746, 'eval_accuracy': 0.6019900497512438, 'eval_runtime': 141.0378, 'eval_samples_per_second': 1.425, 'eval_steps_per_second': 1.425, 'eval_rewards/margins': 1.284481406211853, 'eval_rewards/chosen': -5.516900539398193, 'eval_rewards/rejected': -6.8013811111450195, 'eval_logps/margins': 7.795034408569336, 'eval_logps/chosen': -13.403103828430176, 'eval_logps/rejected': -21.19814109802246, 'epoch': 1.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 2720/5424 [52:14<3:24:11,  4.53s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.5794, 'grad_norm': 4.039405822753906, 'learning_rate': 4.9972487390343625e-06, 'rewards/margins': 1.7455599308013916, 'rewards/chosen': -4.217751502990723, 'rewards/rejected': -5.963311195373535, 'logps/margins': 9.153647422790527, 'logps/chosen': -9.010757446289062, 'logps/rejected': -18.164405822753906, 'accuracy': 0.699999988079071, 'epoch': 1.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 2730/5424 [52:22<39:58,  1.12it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.3493, 'grad_norm': 2.486370801925659, 'learning_rate': 4.968288308296213e-06, 'rewards/margins': 2.5475473403930664, 'rewards/chosen': -4.0790300369262695, 'rewards/rejected': -6.626577854156494, 'logps/margins': 12.262055397033691, 'logps/chosen': -8.005793571472168, 'logps/rejected': -20.267847061157227, 'accuracy': 0.800000011920929, 'epoch': 1.51}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 2740/5424 [52:30<34:35,  1.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 7.9816, 'grad_norm': 0.7175667881965637, 'learning_rate': 4.939328941442838e-06, 'rewards/margins': 4.938241958618164, 'rewards/chosen': -4.153270721435547, 'rewards/rejected': -9.091512680053711, 'logps/margins': 23.83755111694336, 'logps/chosen': -7.578669548034668, 'logps/rejected': -31.41621971130371, 'accuracy': 0.800000011920929, 'epoch': 1.52}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 2750/5424 [52:38<38:53,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.4539, 'grad_norm': 10.462246894836426, 'learning_rate': 4.910371610022204e-06, 'rewards/margins': 2.7209088802337646, 'rewards/chosen': -5.640377044677734, 'rewards/rejected': -8.361286163330078, 'logps/margins': 11.694291114807129, 'logps/chosen': -14.55607795715332, 'logps/rejected': -26.250370025634766, 'accuracy': 0.699999988079071, 'epoch': 1.52}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 2760/5424 [52:46<36:53,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.6616, 'grad_norm': 7.686376571655273, 'learning_rate': 4.881417285513986e-06, 'rewards/margins': 0.22521880269050598, 'rewards/chosen': -6.663047790527344, 'rewards/rejected': -6.888266563415527, 'logps/margins': 3.511431932449341, 'logps/chosen': -16.909696578979492, 'logps/rejected': -20.421127319335938, 'accuracy': 0.6000000238418579, 'epoch': 1.53}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 2770/5424 [52:54<35:43,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 32.763, 'grad_norm': 9.353602409362793, 'learning_rate': 4.852466939296985e-06, 'rewards/margins': -2.219259023666382, 'rewards/chosen': -8.802055358886719, 'rewards/rejected': -6.582797050476074, 'logps/margins': -9.67963695526123, 'logps/chosen': -28.899105072021484, 'logps/rejected': -19.219467163085938, 'accuracy': 0.5, 'epoch': 1.53}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████▏    | 2780/5424 [53:03<38:42,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.6855, 'grad_norm': 7.771176338195801, 'learning_rate': 4.823521542616538e-06, 'rewards/margins': 1.3833982944488525, 'rewards/chosen': -5.413016319274902, 'rewards/rejected': -6.796414852142334, 'logps/margins': 7.637178897857666, 'logps/chosen': -13.436605453491211, 'logps/rejected': -21.07378578186035, 'accuracy': 0.6000000238418579, 'epoch': 1.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████▏    | 2790/5424 [53:12<40:39,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 22.1444, 'grad_norm': 3.0528876781463623, 'learning_rate': 4.7945820665519256e-06, 'rewards/margins': 1.699005126953125, 'rewards/chosen': -7.699929237365723, 'rewards/rejected': -9.398933410644531, 'logps/margins': 10.082616806030273, 'logps/chosen': -20.584877014160156, 'logps/rejected': -30.667491912841797, 'accuracy': 0.6000000238418579, 'epoch': 1.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 2800/5424 [53:20<34:58,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.1842, 'grad_norm': 2.7724132537841797, 'learning_rate': 4.765649481983801e-06, 'rewards/margins': 4.562651634216309, 'rewards/chosen': -4.2652387619018555, 'rewards/rejected': -8.827890396118164, 'logps/margins': 23.77228355407715, 'logps/chosen': -7.814711093902588, 'logps/rejected': -31.58699607849121, 'accuracy': 0.8999999761581421, 'epoch': 1.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 2810/5424 [53:28<37:26,  1.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.8664, 'grad_norm': 5.5035881996154785, 'learning_rate': 4.73672475956162e-06, 'rewards/margins': 2.604630947113037, 'rewards/chosen': -4.8104352951049805, 'rewards/rejected': -7.415065765380859, 'logps/margins': 13.072802543640137, 'logps/chosen': -10.528871536254883, 'logps/rejected': -23.601673126220703, 'accuracy': 0.800000011920929, 'epoch': 1.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 2820/5424 [53:36<34:22,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.5986, 'grad_norm': 3.585066080093384, 'learning_rate': 4.707808869671072e-06, 'rewards/margins': 0.2243824452161789, 'rewards/chosen': -6.217094898223877, 'rewards/rejected': -6.441478729248047, 'logps/margins': 1.008771300315857, 'logps/chosen': -16.57616424560547, 'logps/rejected': -17.58493423461914, 'accuracy': 0.4000000059604645, 'epoch': 1.56}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 2830/5424 [53:45<38:14,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.4079, 'grad_norm': 6.6261372566223145, 'learning_rate': 4.6789027824015295e-06, 'rewards/margins': 0.6172918677330017, 'rewards/chosen': -5.427480220794678, 'rewards/rejected': -6.044772624969482, 'logps/margins': 6.660400390625, 'logps/chosen': -12.458480834960938, 'logps/rejected': -19.11888313293457, 'accuracy': 0.5, 'epoch': 1.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 2840/5424 [53:53<34:31,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.2009, 'grad_norm': 1.362548589706421, 'learning_rate': 4.6500074675134955e-06, 'rewards/margins': 2.3529067039489746, 'rewards/chosen': -5.54088020324707, 'rewards/rejected': -7.893786430358887, 'logps/margins': 15.267278671264648, 'logps/chosen': -11.85499382019043, 'logps/rejected': -27.12227439880371, 'accuracy': 0.8999999761581421, 'epoch': 1.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 2850/5424 [54:01<34:35,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.9669, 'grad_norm': 3.8672032356262207, 'learning_rate': 4.621123894406075e-06, 'rewards/margins': 2.030611991882324, 'rewards/chosen': -3.900286912918091, 'rewards/rejected': -5.930898189544678, 'logps/margins': 8.905229568481445, 'logps/chosen': -9.090097427368164, 'logps/rejected': -17.99532699584961, 'accuracy': 0.699999988079071, 'epoch': 1.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 2860/5424 [54:09<36:04,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.5092, 'grad_norm': 3.4160799980163574, 'learning_rate': 4.592253032084456e-06, 'rewards/margins': 1.634002685546875, 'rewards/chosen': -6.317195892333984, 'rewards/rejected': -7.951199531555176, 'logps/margins': 10.530070304870605, 'logps/chosen': -15.4900484085083, 'logps/rejected': -26.02012062072754, 'accuracy': 0.5, 'epoch': 1.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 2870/5424 [54:17<34:17,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.8966, 'grad_norm': 3.5064756870269775, 'learning_rate': 4.563395849127394e-06, 'rewards/margins': 2.133967638015747, 'rewards/chosen': -5.462055683135986, 'rewards/rejected': -7.596022129058838, 'logps/margins': 12.143829345703125, 'logps/chosen': -13.40434455871582, 'logps/rejected': -25.548173904418945, 'accuracy': 0.800000011920929, 'epoch': 1.59}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 2880/5424 [54:25<33:58,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.0745, 'grad_norm': 3.883646011352539, 'learning_rate': 4.534553313654718e-06, 'rewards/margins': 1.2184953689575195, 'rewards/chosen': -4.749621391296387, 'rewards/rejected': -5.968116760253906, 'logps/margins': 8.988849639892578, 'logps/chosen': -10.018885612487793, 'logps/rejected': -19.007732391357422, 'accuracy': 0.6000000238418579, 'epoch': 1.59}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 2890/5424 [54:34<34:29,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.292, 'grad_norm': 2.5040698051452637, 'learning_rate': 4.505726393294857e-06, 'rewards/margins': 0.8099802136421204, 'rewards/chosen': -5.580935478210449, 'rewards/rejected': -6.390915870666504, 'logps/margins': 5.380999565124512, 'logps/chosen': -12.843835830688477, 'logps/rejected': -18.224834442138672, 'accuracy': 0.5, 'epoch': 1.6}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 2900/5424 [54:42<34:13,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.3417, 'grad_norm': 3.9111011028289795, 'learning_rate': 4.476916055152372e-06, 'rewards/margins': 2.6484463214874268, 'rewards/chosen': -5.023716449737549, 'rewards/rejected': -7.672163486480713, 'logps/margins': 12.880025863647461, 'logps/chosen': -11.740890502929688, 'logps/rejected': -24.62091827392578, 'accuracy': 0.699999988079071, 'epoch': 1.6}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▎    | 2910/5424 [54:50<33:50,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.5524, 'grad_norm': 5.404982089996338, 'learning_rate': 4.448123265775515e-06, 'rewards/margins': -0.12372088432312012, 'rewards/chosen': -6.376626491546631, 'rewards/rejected': -6.25290584564209, 'logps/margins': 2.6360175609588623, 'logps/chosen': -16.61437225341797, 'logps/rejected': -19.250391006469727, 'accuracy': 0.6000000238418579, 'epoch': 1.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 2920/5424 [54:58<35:14,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.0832, 'grad_norm': 2.7677788734436035, 'learning_rate': 4.419348991123796e-06, 'rewards/margins': 3.3333802223205566, 'rewards/chosen': -5.411534309387207, 'rewards/rejected': -8.744913101196289, 'logps/margins': 18.41219711303711, 'logps/chosen': -11.625425338745117, 'logps/rejected': -30.037622451782227, 'accuracy': 0.800000011920929, 'epoch': 1.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 2930/5424 [55:07<32:36,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.8072, 'grad_norm': 3.302442789077759, 'learning_rate': 4.390594196535585e-06, 'rewards/margins': -0.1562773734331131, 'rewards/chosen': -5.956531524658203, 'rewards/rejected': -5.800253391265869, 'logps/margins': 1.9066814184188843, 'logps/chosen': -14.58317756652832, 'logps/rejected': -16.489858627319336, 'accuracy': 0.30000001192092896, 'epoch': 1.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 2940/5424 [55:15<33:00,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.3909, 'grad_norm': 4.650947093963623, 'learning_rate': 4.361859846695722e-06, 'rewards/margins': 0.02528100088238716, 'rewards/chosen': -5.427308082580566, 'rewards/rejected': -5.45258903503418, 'logps/margins': 0.38479310274124146, 'logps/chosen': -13.130884170532227, 'logps/rejected': -13.515676498413086, 'accuracy': 0.5, 'epoch': 1.63}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 2950/5424 [55:23<32:57,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.4955, 'grad_norm': 10.113097190856934, 'learning_rate': 4.33314690560315e-06, 'rewards/margins': 1.8731034994125366, 'rewards/chosen': -6.044100761413574, 'rewards/rejected': -7.9172043800354, 'logps/margins': 9.037703514099121, 'logps/chosen': -16.359060287475586, 'logps/rejected': -25.396764755249023, 'accuracy': 0.699999988079071, 'epoch': 1.63}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▍    | 2960/5424 [55:31<34:42,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.954, 'grad_norm': 6.2399797439575195, 'learning_rate': 4.304456336538579e-06, 'rewards/margins': 2.163999319076538, 'rewards/chosen': -5.075313568115234, 'rewards/rejected': -7.239312171936035, 'logps/margins': 11.23537540435791, 'logps/chosen': -10.517179489135742, 'logps/rejected': -21.752552032470703, 'accuracy': 0.699999988079071, 'epoch': 1.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▍    | 2970/5424 [55:39<34:27,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.124, 'grad_norm': 5.265670299530029, 'learning_rate': 4.275789102032167e-06, 'rewards/margins': 2.903256893157959, 'rewards/chosen': -3.995023012161255, 'rewards/rejected': -6.898279666900635, 'logps/margins': 12.932459831237793, 'logps/chosen': -8.896232604980469, 'logps/rejected': -21.828693389892578, 'accuracy': 0.8999999761581421, 'epoch': 1.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▍    | 2980/5424 [55:47<32:42,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.2997, 'grad_norm': 5.339648246765137, 'learning_rate': 4.247146163831229e-06, 'rewards/margins': 1.8290847539901733, 'rewards/chosen': -4.687860012054443, 'rewards/rejected': -6.516944885253906, 'logps/margins': 9.391592025756836, 'logps/chosen': -10.314074516296387, 'logps/rejected': -19.705665588378906, 'accuracy': 0.6000000238418579, 'epoch': 1.65}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 2990/5424 [55:55<32:49,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.2756, 'grad_norm': 4.195973873138428, 'learning_rate': 4.2185284828679726e-06, 'rewards/margins': 2.1004717350006104, 'rewards/chosen': -4.080248832702637, 'rewards/rejected': -6.180720806121826, 'logps/margins': 11.863288879394531, 'logps/chosen': -8.42533016204834, 'logps/rejected': -20.288619995117188, 'accuracy': 0.699999988079071, 'epoch': 1.65}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 3000/5424 [56:04<33:07,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.0022, 'grad_norm': 2.5582263469696045, 'learning_rate': 4.189937019227257e-06, 'rewards/margins': 0.5104316473007202, 'rewards/chosen': -6.248169422149658, 'rewards/rejected': -6.75860071182251, 'logps/margins': 0.6329530477523804, 'logps/chosen': -18.272470474243164, 'logps/rejected': -18.905426025390625, 'accuracy': 0.5, 'epoch': 1.66}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 3010/5424 [56:12<35:41,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 28.0247, 'grad_norm': 5.407680988311768, 'learning_rate': 4.161372732114388e-06, 'rewards/margins': 2.8069300651550293, 'rewards/chosen': -9.047691345214844, 'rewards/rejected': -11.854621887207031, 'logps/margins': 13.779980659484863, 'logps/chosen': -25.63509750366211, 'logps/rejected': -39.415077209472656, 'accuracy': 0.699999988079071, 'epoch': 1.66}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 3020/5424 [56:20<32:29,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.2575, 'grad_norm': 7.243239402770996, 'learning_rate': 4.13283657982293e-06, 'rewards/margins': 2.102902412414551, 'rewards/chosen': -5.652683734893799, 'rewards/rejected': -7.75558614730835, 'logps/margins': 8.716277122497559, 'logps/chosen': -15.953012466430664, 'logps/rejected': -24.669286727905273, 'accuracy': 0.6000000238418579, 'epoch': 1.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 3030/5424 [56:28<32:27,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.3472, 'grad_norm': 4.800967693328857, 'learning_rate': 4.104329519702568e-06, 'rewards/margins': 2.9027771949768066, 'rewards/chosen': -4.98476505279541, 'rewards/rejected': -7.8875412940979, 'logps/margins': 15.67485237121582, 'logps/chosen': -10.695225715637207, 'logps/rejected': -26.370080947875977, 'accuracy': 0.5, 'epoch': 1.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 3040/5424 [56:36<31:10,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.6136, 'grad_norm': 2.5055124759674072, 'learning_rate': 4.0758525081269816e-06, 'rewards/margins': 1.2675338983535767, 'rewards/chosen': -5.503435134887695, 'rewards/rejected': -6.770968437194824, 'logps/margins': 6.2762908935546875, 'logps/chosen': -14.816305160522461, 'logps/rejected': -21.09259796142578, 'accuracy': 0.699999988079071, 'epoch': 1.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 3050/5424 [56:44<31:30,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.2443, 'grad_norm': 1.9502702951431274, 'learning_rate': 4.04740650046176e-06, 'rewards/margins': 2.1575870513916016, 'rewards/chosen': -4.439446449279785, 'rewards/rejected': -6.597033500671387, 'logps/margins': 10.453277587890625, 'logps/chosen': -8.808351516723633, 'logps/rejected': -19.261629104614258, 'accuracy': 0.800000011920929, 'epoch': 1.69}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▋    | 3060/5424 [56:53<35:24,  1.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.0453, 'grad_norm': 3.571869373321533, 'learning_rate': 4.018992451032353e-06, 'rewards/margins': 3.2888476848602295, 'rewards/chosen': -7.138951301574707, 'rewards/rejected': -10.427799224853516, 'logps/margins': 15.511820793151855, 'logps/chosen': -18.278818130493164, 'logps/rejected': -33.79063415527344, 'accuracy': 0.6000000238418579, 'epoch': 1.69}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 3070/5424 [57:01<33:01,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.2266, 'grad_norm': 6.910975456237793, 'learning_rate': 3.990611313092057e-06, 'rewards/margins': 2.6610050201416016, 'rewards/chosen': -5.788884162902832, 'rewards/rejected': -8.449889183044434, 'logps/margins': 12.71734619140625, 'logps/chosen': -12.564338684082031, 'logps/rejected': -25.28168487548828, 'accuracy': 0.6000000238418579, 'epoch': 1.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 3080/5424 [57:09<31:16,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.501, 'grad_norm': 5.797295093536377, 'learning_rate': 3.962264038790032e-06, 'rewards/margins': -0.10147829353809357, 'rewards/chosen': -5.332039833068848, 'rewards/rejected': -5.23056173324585, 'logps/margins': 0.19327425956726074, 'logps/chosen': -13.322668075561523, 'logps/rejected': -13.515945434570312, 'accuracy': 0.6000000238418579, 'epoch': 1.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 3090/5424 [57:18<31:23,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.5415, 'grad_norm': 1.1072380542755127, 'learning_rate': 3.933951579139354e-06, 'rewards/margins': 3.3623719215393066, 'rewards/chosen': -4.049736976623535, 'rewards/rejected': -7.412107944488525, 'logps/margins': 14.418354988098145, 'logps/chosen': -7.8731560707092285, 'logps/rejected': -22.291513442993164, 'accuracy': 0.699999988079071, 'epoch': 1.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 3100/5424 [57:26<31:11,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.5868, 'grad_norm': 1.5256075859069824, 'learning_rate': 3.905674883985118e-06, 'rewards/margins': -0.8630625009536743, 'rewards/chosen': -5.718475341796875, 'rewards/rejected': -4.85541296005249, 'logps/margins': -3.224799633026123, 'logps/chosen': -15.813932418823242, 'logps/rejected': -12.589132308959961, 'accuracy': 0.4000000059604645, 'epoch': 1.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 3110/5424 [57:34<31:39,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.2464, 'grad_norm': 7.733896255493164, 'learning_rate': 3.877434901972566e-06, 'rewards/margins': 1.6863336563110352, 'rewards/chosen': -5.231781959533691, 'rewards/rejected': -6.918116092681885, 'logps/margins': 7.150366306304932, 'logps/chosen': -13.207295417785645, 'logps/rejected': -20.357662200927734, 'accuracy': 0.6000000238418579, 'epoch': 1.72}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 3120/5424 [57:42<29:46,  1.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.9903, 'grad_norm': 2.9353363513946533, 'learning_rate': 3.849232580515269e-06, 'rewards/margins': 1.0966384410858154, 'rewards/chosen': -5.6528143882751465, 'rewards/rejected': -6.749453067779541, 'logps/margins': 6.976377010345459, 'logps/chosen': -13.389492988586426, 'logps/rejected': -20.365869522094727, 'accuracy': 0.6000000238418579, 'epoch': 1.73}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 3130/5424 [57:50<30:04,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.7986, 'grad_norm': 3.6190383434295654, 'learning_rate': 3.821068865763326e-06, 'rewards/margins': 1.0260999202728271, 'rewards/chosen': -4.499231815338135, 'rewards/rejected': -5.525331974029541, 'logps/margins': 7.689568996429443, 'logps/chosen': -9.88046932220459, 'logps/rejected': -17.570035934448242, 'accuracy': 0.5, 'epoch': 1.73}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 3140/5424 [57:58<30:31,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.1565, 'grad_norm': 3.3841238021850586, 'learning_rate': 3.792944702571641e-06, 'rewards/margins': 1.8161178827285767, 'rewards/chosen': -4.358348369598389, 'rewards/rejected': -6.174466133117676, 'logps/margins': 8.494229316711426, 'logps/chosen': -10.322835922241211, 'logps/rejected': -18.817066192626953, 'accuracy': 0.800000011920929, 'epoch': 1.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 3150/5424 [58:06<31:45,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.5339, 'grad_norm': 7.3650431632995605, 'learning_rate': 3.7648610344682153e-06, 'rewards/margins': 0.8482305407524109, 'rewards/chosen': -6.538651943206787, 'rewards/rejected': -7.386883735656738, 'logps/margins': 4.035968780517578, 'logps/chosen': -18.585063934326172, 'logps/rejected': -22.621034622192383, 'accuracy': 0.4000000059604645, 'epoch': 1.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 3160/5424 [58:14<30:01,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.829, 'grad_norm': 3.227108955383301, 'learning_rate': 3.736818803622492e-06, 'rewards/margins': 0.908416748046875, 'rewards/chosen': -5.3617777824401855, 'rewards/rejected': -6.270195007324219, 'logps/margins': 5.648860931396484, 'logps/chosen': -11.07627010345459, 'logps/rejected': -16.725128173828125, 'accuracy': 0.800000011920929, 'epoch': 1.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 3170/5424 [58:22<30:06,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.4376, 'grad_norm': 3.5372843742370605, 'learning_rate': 3.708818950813751e-06, 'rewards/margins': 2.187620162963867, 'rewards/chosen': -4.7259931564331055, 'rewards/rejected': -6.913613319396973, 'logps/margins': 8.919021606445312, 'logps/chosen': -12.617979049682617, 'logps/rejected': -21.536998748779297, 'accuracy': 0.699999988079071, 'epoch': 1.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▊    | 3180/5424 [58:30<29:47,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.5781, 'grad_norm': 5.617955684661865, 'learning_rate': 3.6808624153995432e-06, 'rewards/margins': 1.6143662929534912, 'rewards/chosen': -4.896588325500488, 'rewards/rejected': -6.510954856872559, 'logps/margins': 7.8427581787109375, 'logps/chosen': -10.869415283203125, 'logps/rejected': -18.712169647216797, 'accuracy': 0.800000011920929, 'epoch': 1.76}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 3190/5424 [58:39<31:18,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.3182, 'grad_norm': 5.293271064758301, 'learning_rate': 3.6557393451075337e-06, 'rewards/margins': nan, 'rewards/chosen': nan, 'rewards/rejected': -6.25095272064209, 'logps/margins': nan, 'logps/chosen': nan, 'logps/rejected': -17.022153854370117, 'accuracy': 0.5, 'epoch': 1.76}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 3200/5424 [58:47<30:59,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.126, 'grad_norm': 3.1401665210723877, 'learning_rate': 3.6278676954428634e-06, 'rewards/margins': -0.18533630669116974, 'rewards/chosen': -6.010137557983398, 'rewards/rejected': -5.824801445007324, 'logps/margins': 2.5005269050598145, 'logps/chosen': -15.506521224975586, 'logps/rejected': -18.00704574584961, 'accuracy': 0.4000000059604645, 'epoch': 1.77}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 3210/5424 [58:55<29:54,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 7.8726, 'grad_norm': 1.795722246170044, 'learning_rate': 3.600042078978906e-06, 'rewards/margins': 4.409274101257324, 'rewards/chosen': -4.318619728088379, 'rewards/rejected': -8.727893829345703, 'logps/margins': 21.519197463989258, 'logps/chosen': -7.599053859710693, 'logps/rejected': -29.118249893188477, 'accuracy': 0.800000011920929, 'epoch': 1.78}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 3220/5424 [59:03<31:12,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.7572, 'grad_norm': 5.271749973297119, 'learning_rate': 3.5722634292278136e-06, 'rewards/margins': 0.502513587474823, 'rewards/chosen': -6.564212799072266, 'rewards/rejected': -7.0667266845703125, 'logps/margins': 2.2838053703308105, 'logps/chosen': -17.10704803466797, 'logps/rejected': -19.390851974487305, 'accuracy': 0.5, 'epoch': 1.78}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████▉    | 3230/5424 [59:11<29:44,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.4154, 'grad_norm': 7.728300094604492, 'learning_rate': 3.5445326781260756e-06, 'rewards/margins': 1.4139541387557983, 'rewards/chosen': -6.339304447174072, 'rewards/rejected': -7.75325870513916, 'logps/margins': 9.815739631652832, 'logps/chosen': -15.641782760620117, 'logps/rejected': -25.457523345947266, 'accuracy': 0.5, 'epoch': 1.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████▉    | 3240/5424 [59:19<28:39,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.6329, 'grad_norm': 9.154850006103516, 'learning_rate': 3.5168507560032457e-06, 'rewards/margins': -0.85755854845047, 'rewards/chosen': -5.565855979919434, 'rewards/rejected': -4.708296775817871, 'logps/margins': -0.5015007257461548, 'logps/chosen': -13.923550605773926, 'logps/rejected': -13.422048568725586, 'accuracy': 0.5, 'epoch': 1.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████▉    | 3250/5424 [59:28<29:05,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.5366, 'grad_norm': 2.997694969177246, 'learning_rate': 3.4892185915507266e-06, 'rewards/margins': 0.3973403573036194, 'rewards/chosen': -6.060560703277588, 'rewards/rejected': -6.457901954650879, 'logps/margins': 3.3349967002868652, 'logps/chosen': -16.632686614990234, 'logps/rejected': -19.967683792114258, 'accuracy': 0.6000000238418579, 'epoch': 1.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 3260/5424 [59:36<29:13,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.1284, 'grad_norm': 6.671473979949951, 'learning_rate': 3.461637111790617e-06, 'rewards/margins': 0.09604072570800781, 'rewards/chosen': -5.76905632019043, 'rewards/rejected': -5.865096092224121, 'logps/margins': 1.1988376379013062, 'logps/chosen': -15.974408149719238, 'logps/rejected': -17.17324447631836, 'accuracy': 0.5, 'epoch': 1.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 3270/5424 [59:44<28:19,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.7096, 'grad_norm': 3.632866144180298, 'learning_rate': 3.434107242044617e-06, 'rewards/margins': 2.3887596130371094, 'rewards/chosen': -4.427089214324951, 'rewards/rejected': -6.815848350524902, 'logps/margins': 13.005708694458008, 'logps/chosen': -9.940607070922852, 'logps/rejected': -22.94631576538086, 'accuracy': 0.6000000238418579, 'epoch': 1.81}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 3280/5424 [59:52<28:58,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.4274, 'grad_norm': 4.981480598449707, 'learning_rate': 3.406629905902978e-06, 'rewards/margins': 1.7766673564910889, 'rewards/chosen': -5.132824420928955, 'rewards/rejected': -6.909491539001465, 'logps/margins': 7.487724304199219, 'logps/chosen': -13.593002319335938, 'logps/rejected': -21.080726623535156, 'accuracy': 0.6000000238418579, 'epoch': 1.81}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 3290/5424 [1:00:00<27:49,  1.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 7.02, 'grad_norm': 1.4785758256912231, 'learning_rate': 3.379206025193519e-06, 'rewards/margins': 3.084792375564575, 'rewards/chosen': -4.074808120727539, 'rewards/rejected': -7.159600734710693, 'logps/margins': 16.172391891479492, 'logps/chosen': -6.757907867431641, 'logps/rejected': -22.930295944213867, 'accuracy': 0.8999999761581421, 'epoch': 1.82}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 3300/5424 [1:00:08<28:02,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.2091, 'grad_norm': 0.8513771295547485, 'learning_rate': 3.351836519950702e-06, 'rewards/margins': 2.4638214111328125, 'rewards/chosen': -4.58004903793335, 'rewards/rejected': -7.043870449066162, 'logps/margins': 12.894769668579102, 'logps/chosen': -10.952276229858398, 'logps/rejected': -23.8470458984375, 'accuracy': 0.8999999761581421, 'epoch': 1.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 3310/5424 [1:00:16<28:50,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.2134, 'grad_norm': 4.943087577819824, 'learning_rate': 3.324522308384759e-06, 'rewards/margins': 1.1888039112091064, 'rewards/chosen': -6.380387306213379, 'rewards/rejected': -7.56919002532959, 'logps/margins': 5.0027174949646, 'logps/chosen': -17.211811065673828, 'logps/rejected': -22.214529037475586, 'accuracy': 0.6000000238418579, 'epoch': 1.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 3320/5424 [1:00:25<28:45,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.8161, 'grad_norm': 2.1484858989715576, 'learning_rate': 3.2972643068509014e-06, 'rewards/margins': 1.7697166204452515, 'rewards/chosen': -4.2582268714904785, 'rewards/rejected': -6.0279436111450195, 'logps/margins': 7.030369758605957, 'logps/chosen': -9.269376754760742, 'logps/rejected': -16.299745559692383, 'accuracy': 0.699999988079071, 'epoch': 1.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████▏   | 3330/5424 [1:00:33<28:24,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.1605, 'grad_norm': 2.95133376121521, 'learning_rate': 3.270063429818566e-06, 'rewards/margins': -0.012200379744172096, 'rewards/chosen': -5.489011287689209, 'rewards/rejected': -5.476810932159424, 'logps/margins': -0.5922945141792297, 'logps/chosen': -14.735311508178711, 'logps/rejected': -14.143017768859863, 'accuracy': 0.5, 'epoch': 1.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 3340/5424 [1:00:41<27:47,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.0716, 'grad_norm': 3.326308250427246, 'learning_rate': 3.2429205898407403e-06, 'rewards/margins': 2.399960517883301, 'rewards/chosen': -4.915163516998291, 'rewards/rejected': -7.315123558044434, 'logps/margins': 15.237482070922852, 'logps/chosen': -10.176763534545898, 'logps/rejected': -25.41424560546875, 'accuracy': 0.6000000238418579, 'epoch': 1.85}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 3350/5424 [1:00:49<27:01,  1.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.6763, 'grad_norm': 2.0451343059539795, 'learning_rate': 3.2158366975233412e-06, 'rewards/margins': -0.03471650928258896, 'rewards/chosen': -5.737347602844238, 'rewards/rejected': -5.70263147354126, 'logps/margins': -0.22089090943336487, 'logps/chosen': -16.751157760620117, 'logps/rejected': -16.5302677154541, 'accuracy': 0.4000000059604645, 'epoch': 1.85}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 3360/5424 [1:00:57<28:02,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.4932, 'grad_norm': 3.777411937713623, 'learning_rate': 3.1888126614946757e-06, 'rewards/margins': 0.8917387127876282, 'rewards/chosen': -5.480238914489746, 'rewards/rejected': -6.37197732925415, 'logps/margins': 5.701801776885986, 'logps/chosen': -13.150212287902832, 'logps/rejected': -18.852014541625977, 'accuracy': 0.800000011920929, 'epoch': 1.86}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 3370/5424 [1:01:05<28:12,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.8219, 'grad_norm': 3.2299001216888428, 'learning_rate': 3.161849388374952e-06, 'rewards/margins': 2.0821785926818848, 'rewards/chosen': -5.861297607421875, 'rewards/rejected': -7.94347620010376, 'logps/margins': 7.286296844482422, 'logps/chosen': -16.035640716552734, 'logps/rejected': -23.321941375732422, 'accuracy': 0.5, 'epoch': 1.86}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 3380/5424 [1:01:14<27:16,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.376, 'grad_norm': 6.817086696624756, 'learning_rate': 3.134947782745865e-06, 'rewards/margins': 1.4293174743652344, 'rewards/chosen': -5.911830902099609, 'rewards/rejected': -7.341148376464844, 'logps/margins': 7.6681084632873535, 'logps/chosen': -14.657849311828613, 'logps/rejected': -22.325956344604492, 'accuracy': 0.699999988079071, 'epoch': 1.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▎   | 3390/5424 [1:01:21<27:17,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.352, 'grad_norm': 5.0414323806762695, 'learning_rate': 3.108108747120242e-06, 'rewards/margins': 0.7867997884750366, 'rewards/chosen': -5.058559417724609, 'rewards/rejected': -5.845358848571777, 'logps/margins': 5.7980828285217285, 'logps/chosen': -12.309808731079102, 'logps/rejected': -18.107891082763672, 'accuracy': 0.699999988079071, 'epoch': 1.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 3400/5424 [1:01:29<26:43,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 7.9087, 'grad_norm': 3.694606065750122, 'learning_rate': 3.0813331819117785e-06, 'rewards/margins': 2.5688493251800537, 'rewards/chosen': -3.9033761024475098, 'rewards/rejected': -6.472225189208984, 'logps/margins': 12.509284973144531, 'logps/chosen': -7.638268947601318, 'logps/rejected': -20.147554397583008, 'accuracy': 0.800000011920929, 'epoch': 1.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 3410/5424 [1:01:37<26:42,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.1254, 'grad_norm': 3.2768685817718506, 'learning_rate': 3.0546219854048152e-06, 'rewards/margins': -0.2653328478336334, 'rewards/chosen': -5.97866153717041, 'rewards/rejected': -5.713329315185547, 'logps/margins': 0.014279365539550781, 'logps/chosen': -15.256757736206055, 'logps/rejected': -15.271036148071289, 'accuracy': 0.4000000059604645, 'epoch': 1.89}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 3420/5424 [1:01:46<27:13,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.2349, 'grad_norm': 2.872908115386963, 'learning_rate': 3.0279760537242177e-06, 'rewards/margins': 0.0865660160779953, 'rewards/chosen': -5.546431064605713, 'rewards/rejected': -5.632996559143066, 'logps/margins': -1.4886763095855713, 'logps/chosen': -16.247472763061523, 'logps/rejected': -14.758796691894531, 'accuracy': 0.5, 'epoch': 1.89}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 3430/5424 [1:01:54<26:46,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.5852, 'grad_norm': 3.7063395977020264, 'learning_rate': 3.001396280805293e-06, 'rewards/margins': 1.4591678380966187, 'rewards/chosen': -4.382604122161865, 'rewards/rejected': -5.841772556304932, 'logps/margins': 8.14000129699707, 'logps/chosen': -10.811159133911133, 'logps/rejected': -18.951160430908203, 'accuracy': 0.6000000238418579, 'epoch': 1.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 3440/5424 [1:02:02<26:56,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.618, 'grad_norm': 4.67020320892334, 'learning_rate': 2.9748835583638197e-06, 'rewards/margins': 3.8232669830322266, 'rewards/chosen': -5.2113037109375, 'rewards/rejected': -9.034570693969727, 'logps/margins': 19.71817970275879, 'logps/chosen': -11.995610237121582, 'logps/rejected': -31.713787078857422, 'accuracy': 0.800000011920929, 'epoch': 1.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▎   | 3450/5424 [1:02:10<27:25,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.4176, 'grad_norm': 3.659306287765503, 'learning_rate': 2.9484387758661193e-06, 'rewards/margins': 2.121340274810791, 'rewards/chosen': -5.946310520172119, 'rewards/rejected': -8.06765079498291, 'logps/margins': 11.160379409790039, 'logps/chosen': -13.875958442687988, 'logps/rejected': -25.036334991455078, 'accuracy': 0.8999999761581421, 'epoch': 1.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 3460/5424 [1:02:18<27:58,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.0275, 'grad_norm': 11.724209785461426, 'learning_rate': 2.922062820499218e-06, 'rewards/margins': 0.7147854566574097, 'rewards/chosen': -6.7601447105407715, 'rewards/rejected': -7.474931240081787, 'logps/margins': 5.900176048278809, 'logps/chosen': -18.131183624267578, 'logps/rejected': -24.031360626220703, 'accuracy': 0.6000000238418579, 'epoch': 1.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 3470/5424 [1:02:27<26:06,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.3256, 'grad_norm': 2.0652425289154053, 'learning_rate': 2.895756577141088e-06, 'rewards/margins': 1.3302403688430786, 'rewards/chosen': -5.058150768280029, 'rewards/rejected': -6.388391017913818, 'logps/margins': 4.705951690673828, 'logps/chosen': -13.667033195495605, 'logps/rejected': -18.37298583984375, 'accuracy': 0.699999988079071, 'epoch': 1.92}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 3480/5424 [1:02:35<26:34,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.5738, 'grad_norm': 8.045490264892578, 'learning_rate': 2.869520928330949e-06, 'rewards/margins': 2.1325244903564453, 'rewards/chosen': -5.420376777648926, 'rewards/rejected': -7.552901268005371, 'logps/margins': 10.151117324829102, 'logps/chosen': -13.074508666992188, 'logps/rejected': -23.22562599182129, 'accuracy': 0.6000000238418579, 'epoch': 1.92}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 3490/5424 [1:02:43<26:00,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.7851, 'grad_norm': 1.2707754373550415, 'learning_rate': 2.8433567542396767e-06, 'rewards/margins': 2.26511812210083, 'rewards/chosen': -4.493438720703125, 'rewards/rejected': -6.758556365966797, 'logps/margins': 9.819757461547852, 'logps/chosen': -10.288317680358887, 'logps/rejected': -20.108076095581055, 'accuracy': 0.800000011920929, 'epoch': 1.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▍   | 3500/5424 [1:02:51<26:09,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.546, 'grad_norm': 3.273289918899536, 'learning_rate': 2.8172649326402624e-06, 'rewards/margins': 2.401864528656006, 'rewards/chosen': -3.8618102073669434, 'rewards/rejected': -6.263674736022949, 'logps/margins': 12.891157150268555, 'logps/chosen': -7.8968305587768555, 'logps/rejected': -20.78799057006836, 'accuracy': 0.6000000238418579, 'epoch': 1.94}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▍   | 3510/5424 [1:02:59<26:15,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.1333, 'grad_norm': 1.33565354347229, 'learning_rate': 2.7912463388783687e-06, 'rewards/margins': 2.469158172607422, 'rewards/chosen': -4.7873005867004395, 'rewards/rejected': -7.256458282470703, 'logps/margins': 11.314844131469727, 'logps/chosen': -10.631570816040039, 'logps/rejected': -21.946414947509766, 'accuracy': 0.800000011920929, 'epoch': 1.94}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▍   | 3520/5424 [1:03:07<25:42,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.8907, 'grad_norm': 4.315189838409424, 'learning_rate': 2.76530184584296e-06, 'rewards/margins': -0.5301070213317871, 'rewards/chosen': -6.090020656585693, 'rewards/rejected': -5.559913635253906, 'logps/margins': -3.317852020263672, 'logps/chosen': -17.365550994873047, 'logps/rejected': -14.047698974609375, 'accuracy': 0.5, 'epoch': 1.95}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 3530/5424 [1:03:16<25:08,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.5094, 'grad_norm': 3.4434423446655273, 'learning_rate': 2.7394323239370247e-06, 'rewards/margins': 3.6706466674804688, 'rewards/chosen': -4.99989128112793, 'rewards/rejected': -8.670537948608398, 'logps/margins': 13.797521591186523, 'logps/chosen': -12.255229949951172, 'logps/rejected': -26.052753448486328, 'accuracy': 0.800000011920929, 'epoch': 1.95}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 3540/5424 [1:03:24<25:47,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.0226, 'grad_norm': 5.605630397796631, 'learning_rate': 2.713638641048364e-06, 'rewards/margins': 0.7513864040374756, 'rewards/chosen': -5.823851585388184, 'rewards/rejected': -6.575238227844238, 'logps/margins': 5.648486137390137, 'logps/chosen': -14.444256782531738, 'logps/rejected': -20.092741012573242, 'accuracy': 0.5, 'epoch': 1.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 3550/5424 [1:03:32<26:08,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.7491, 'grad_norm': 5.1388726234436035, 'learning_rate': 2.68792166252049e-06, 'rewards/margins': 1.1294509172439575, 'rewards/chosen': -5.506186008453369, 'rewards/rejected': -6.635636806488037, 'logps/margins': 7.476065158843994, 'logps/chosen': -14.225626945495605, 'logps/rejected': -21.70169448852539, 'accuracy': 0.6000000238418579, 'epoch': 1.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 3560/5424 [1:03:40<24:44,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 7.0875, 'grad_norm': 4.395471572875977, 'learning_rate': 2.6622822511235787e-06, 'rewards/margins': 2.47414493560791, 'rewards/chosen': -3.657322645187378, 'rewards/rejected': -6.131467819213867, 'logps/margins': 14.701739311218262, 'logps/chosen': -6.677903175354004, 'logps/rejected': -21.379642486572266, 'accuracy': 0.699999988079071, 'epoch': 1.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 3570/5424 [1:03:49<24:54,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.3863, 'grad_norm': 7.645789623260498, 'learning_rate': 2.6367212670255366e-06, 'rewards/margins': 1.3448560237884521, 'rewards/chosen': -6.021161079406738, 'rewards/rejected': -7.3660173416137695, 'logps/margins': 5.610428810119629, 'logps/chosen': -16.1505069732666, 'logps/rejected': -21.760936737060547, 'accuracy': 0.5, 'epoch': 1.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 3580/5424 [1:03:57<25:03,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.0777, 'grad_norm': 4.691607475280762, 'learning_rate': 2.6112395677631396e-06, 'rewards/margins': 1.4785146713256836, 'rewards/chosen': -5.984158515930176, 'rewards/rejected': -7.462672233581543, 'logps/margins': 8.387033462524414, 'logps/chosen': -14.093348503112793, 'logps/rejected': -22.480382919311523, 'accuracy': 0.6000000238418579, 'epoch': 1.98}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 3590/5424 [1:04:05<24:24,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.4273, 'grad_norm': 2.776052236557007, 'learning_rate': 2.585838008213264e-06, 'rewards/margins': 0.35247188806533813, 'rewards/chosen': -5.353411674499512, 'rewards/rejected': -5.705883979797363, 'logps/margins': 2.9555461406707764, 'logps/chosen': -13.859097480773926, 'logps/rejected': -16.81464195251465, 'accuracy': 0.5, 'epoch': 1.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▋   | 3600/5424 [1:04:13<24:06,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.2349, 'grad_norm': 2.7972054481506348, 'learning_rate': 2.5605174405642015e-06, 'rewards/margins': 1.033064603805542, 'rewards/chosen': -4.618227958679199, 'rewards/rejected': -5.651292324066162, 'logps/margins': 4.67417049407959, 'logps/chosen': -10.917524337768555, 'logps/rejected': -15.591692924499512, 'accuracy': 0.5, 'epoch': 1.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 3610/5424 [1:04:21<26:06,  1.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.7852, 'grad_norm': 2.9949986934661865, 'learning_rate': 2.5352787142870783e-06, 'rewards/margins': 1.1116197109222412, 'rewards/chosen': -7.058612823486328, 'rewards/rejected': -8.170232772827148, 'logps/margins': 7.069146156311035, 'logps/chosen': -17.171649932861328, 'logps/rejected': -24.240795135498047, 'accuracy': 0.6000000238418579, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 3620/5424 [1:04:29<27:07,  1.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.9602, 'grad_norm': 4.216025352478027, 'learning_rate': 2.5101226761073525e-06, 'rewards/margins': 0.5435975193977356, 'rewards/chosen': -5.375491142272949, 'rewards/rejected': -5.919088840484619, 'logps/margins': 6.271728038787842, 'logps/chosen': -12.750261306762695, 'logps/rejected': -19.021991729736328, 'accuracy': 0.6000000238418579, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 3630/5424 [1:04:37<24:28,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.2406, 'grad_norm': 3.930049180984497, 'learning_rate': 2.485050169976407e-06, 'rewards/margins': 1.2319843769073486, 'rewards/chosen': -5.439457893371582, 'rewards/rejected': -6.67144250869751, 'logps/margins': 7.7203240394592285, 'logps/chosen': -13.368623733520508, 'logps/rejected': -21.088947296142578, 'accuracy': 0.699999988079071, 'epoch': 2.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 3640/5424 [1:04:46<24:46,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.6463, 'grad_norm': 2.3445446491241455, 'learning_rate': 2.4600620370432345e-06, 'rewards/margins': 2.790214776992798, 'rewards/chosen': -4.846582889556885, 'rewards/rejected': -7.636796474456787, 'logps/margins': 12.63306999206543, 'logps/chosen': -12.005045890808105, 'logps/rejected': -24.63811683654785, 'accuracy': 0.800000011920929, 'epoch': 2.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 3650/5424 [1:04:54<24:44,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.0739, 'grad_norm': 2.6047818660736084, 'learning_rate': 2.435159115626218e-06, 'rewards/margins': 2.218658685684204, 'rewards/chosen': -5.825801849365234, 'rewards/rejected': -8.04446029663086, 'logps/margins': 13.549273490905762, 'logps/chosen': -14.121350288391113, 'logps/rejected': -27.670623779296875, 'accuracy': 0.6000000238418579, 'epoch': 2.02}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 3660/5424 [1:05:02<23:37,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.2857, 'grad_norm': 2.101792812347412, 'learning_rate': 2.4103422411850126e-06, 'rewards/margins': 2.2555630207061768, 'rewards/chosen': -4.469383239746094, 'rewards/rejected': -6.72494649887085, 'logps/margins': 9.12586498260498, 'logps/chosen': -10.466168403625488, 'logps/rejected': -19.592031478881836, 'accuracy': 0.800000011920929, 'epoch': 2.02}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 3670/5424 [1:05:10<23:36,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.0567, 'grad_norm': 4.118183135986328, 'learning_rate': 2.3856122462925125e-06, 'rewards/margins': -0.181649848818779, 'rewards/chosen': -4.72296142578125, 'rewards/rejected': -4.541311740875244, 'logps/margins': 1.4991837739944458, 'logps/chosen': -10.783821105957031, 'logps/rejected': -12.283005714416504, 'accuracy': 0.6000000238418579, 'epoch': 2.03}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 3680/5424 [1:05:18<23:26,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.9892, 'grad_norm': 3.1463088989257812, 'learning_rate': 2.360969960606919e-06, 'rewards/margins': 2.79498291015625, 'rewards/chosen': -4.783488750457764, 'rewards/rejected': -7.5784711837768555, 'logps/margins': 12.958477973937988, 'logps/chosen': -10.639084815979004, 'logps/rejected': -23.59756088256836, 'accuracy': 0.8999999761581421, 'epoch': 2.04}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 3690/5424 [1:05:26<23:41,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.2799, 'grad_norm': 2.4163706302642822, 'learning_rate': 2.3364162108439014e-06, 'rewards/margins': 0.17110705375671387, 'rewards/chosen': -5.265017986297607, 'rewards/rejected': -5.436125755310059, 'logps/margins': 3.739995241165161, 'logps/chosen': -11.913969039916992, 'logps/rejected': -15.653965950012207, 'accuracy': 0.4000000059604645, 'epoch': 2.04}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 3700/5424 [1:05:34<23:05,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.6568, 'grad_norm': 1.6202765703201294, 'learning_rate': 2.311951820748876e-06, 'rewards/margins': 2.652656078338623, 'rewards/chosen': -3.9919464588165283, 'rewards/rejected': -6.644602298736572, 'logps/margins': 12.96825122833252, 'logps/chosen': -9.082487106323242, 'logps/rejected': -22.050737380981445, 'accuracy': 0.800000011920929, 'epoch': 2.05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 3710/5424 [1:05:42<23:06,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.2454, 'grad_norm': 2.8591794967651367, 'learning_rate': 2.287577611069357e-06, 'rewards/margins': 0.48496493697166443, 'rewards/chosen': -5.124701976776123, 'rewards/rejected': -5.609666347503662, 'logps/margins': 4.1652445793151855, 'logps/chosen': -11.925067901611328, 'logps/rejected': -16.090312957763672, 'accuracy': 0.5, 'epoch': 2.05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▊   | 3720/5424 [1:05:51<23:42,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.4045, 'grad_norm': 12.120416641235352, 'learning_rate': 2.2632943995274293e-06, 'rewards/margins': 0.09131638705730438, 'rewards/chosen': -6.2516069412231445, 'rewards/rejected': -6.342923164367676, 'logps/margins': 4.223238468170166, 'logps/chosen': -15.210153579711914, 'logps/rejected': -19.433391571044922, 'accuracy': 0.6000000238418579, 'epoch': 2.06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 3730/5424 [1:05:59<24:23,  1.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 23.5676, 'grad_norm': 5.81541633605957, 'learning_rate': 2.2391030007923063e-06, 'rewards/margins': 0.4641045033931732, 'rewards/chosen': -7.2772979736328125, 'rewards/rejected': -7.741401672363281, 'logps/margins': 1.3456671237945557, 'logps/chosen': -21.309362411499023, 'logps/rejected': -22.655031204223633, 'accuracy': 0.4000000059604645, 'epoch': 2.06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 3740/5424 [1:06:07<22:14,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.6349, 'grad_norm': 2.042142152786255, 'learning_rate': 2.215004226453014e-06, 'rewards/margins': 1.364922285079956, 'rewards/chosen': -4.541468620300293, 'rewards/rejected': -5.906390190124512, 'logps/margins': 6.916679382324219, 'logps/chosen': -10.685328483581543, 'logps/rejected': -17.602008819580078, 'accuracy': 0.6000000238418579, 'epoch': 2.07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 3750/5424 [1:06:16<23:32,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.1013, 'grad_norm': 6.809256076812744, 'learning_rate': 2.190998884991143e-06, 'rewards/margins': 1.064496397972107, 'rewards/chosen': -6.659471035003662, 'rewards/rejected': -7.723967552185059, 'logps/margins': 7.294978141784668, 'logps/chosen': -16.39539909362793, 'logps/rejected': -23.690380096435547, 'accuracy': 0.6000000238418579, 'epoch': 2.07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 3760/5424 [1:06:24<22:52,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.22, 'grad_norm': 3.6018245220184326, 'learning_rate': 2.1670877817537467e-06, 'rewards/margins': 0.7655806541442871, 'rewards/chosen': -4.480027675628662, 'rewards/rejected': -5.245608329772949, 'logps/margins': 5.018751621246338, 'logps/chosen': -9.385339736938477, 'logps/rejected': -14.404092788696289, 'accuracy': 0.6000000238418579, 'epoch': 2.08}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|██████▉   | 3770/5424 [1:06:32<22:13,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.6087, 'grad_norm': 4.329761981964111, 'learning_rate': 2.1432717189263038e-06, 'rewards/margins': 2.0214972496032715, 'rewards/chosen': -5.479802131652832, 'rewards/rejected': -7.501298427581787, 'logps/margins': 9.388895034790039, 'logps/chosen': -14.584895133972168, 'logps/rejected': -23.973791122436523, 'accuracy': 0.6000000238418579, 'epoch': 2.09}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|██████▉   | 3780/5424 [1:06:40<22:05,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.5511, 'grad_norm': 8.165594100952148, 'learning_rate': 2.119551495505819e-06, 'rewards/margins': 2.4855430126190186, 'rewards/chosen': -4.392915725708008, 'rewards/rejected': -6.8784589767456055, 'logps/margins': 12.196743965148926, 'logps/chosen': -9.113550186157227, 'logps/rejected': -21.31029510498047, 'accuracy': 0.699999988079071, 'epoch': 2.09}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|██████▉   | 3790/5424 [1:06:48<21:41,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.1752, 'grad_norm': 1.7465412616729736, 'learning_rate': 2.095927907274014e-06, 'rewards/margins': 1.5798168182373047, 'rewards/chosen': -4.792936325073242, 'rewards/rejected': -6.372752666473389, 'logps/margins': 6.780512809753418, 'logps/chosen': -12.884936332702637, 'logps/rejected': -19.665449142456055, 'accuracy': 0.699999988079071, 'epoch': 2.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 3800/5424 [1:06:56<21:39,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.0453, 'grad_norm': 3.7182137966156006, 'learning_rate': 2.072401746770627e-06, 'rewards/margins': -0.6394577622413635, 'rewards/chosen': -6.389174938201904, 'rewards/rejected': -5.749718189239502, 'logps/margins': -0.375638484954834, 'logps/chosen': -17.173213958740234, 'logps/rejected': -16.797574996948242, 'accuracy': 0.30000001192092896, 'epoch': 2.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 3810/5424 [1:07:04<21:29,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.8741, 'grad_norm': 4.231396198272705, 'learning_rate': 2.0489738032668287e-06, 'rewards/margins': 0.6242597699165344, 'rewards/chosen': -4.9781880378723145, 'rewards/rejected': -5.602447986602783, 'logps/margins': 1.94098699092865, 'logps/chosen': -13.558077812194824, 'logps/rejected': -15.499064445495605, 'accuracy': 0.30000001192092896, 'epoch': 2.11}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 3820/5424 [1:07:12<21:27,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.5567, 'grad_norm': 5.140616416931152, 'learning_rate': 2.0256448627387364e-06, 'rewards/margins': 1.418195366859436, 'rewards/chosen': -4.58632230758667, 'rewards/rejected': -6.004517555236816, 'logps/margins': 6.353632926940918, 'logps/chosen': -12.67219352722168, 'logps/rejected': -19.025827407836914, 'accuracy': 0.699999988079071, 'epoch': 2.11}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 3830/5424 [1:07:20<22:26,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.6115, 'grad_norm': 7.707736015319824, 'learning_rate': 2.0024157078410554e-06, 'rewards/margins': 2.1109583377838135, 'rewards/chosen': -4.7626190185546875, 'rewards/rejected': -6.873577117919922, 'logps/margins': 12.20618724822998, 'logps/chosen': -10.657564163208008, 'logps/rejected': -22.863750457763672, 'accuracy': 0.800000011920929, 'epoch': 2.12}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 3840/5424 [1:07:28<21:13,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.1223, 'grad_norm': 7.801382064819336, 'learning_rate': 1.9792871178808164e-06, 'rewards/margins': 1.5956323146820068, 'rewards/chosen': -5.675603866577148, 'rewards/rejected': -7.271235466003418, 'logps/margins': 3.4645628929138184, 'logps/chosen': -15.981962203979492, 'logps/rejected': -19.44652557373047, 'accuracy': 0.5, 'epoch': 2.12}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 3850/5424 [1:07:36<21:10,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.5008, 'grad_norm': 5.330994129180908, 'learning_rate': 1.956259868791226e-06, 'rewards/margins': 0.24617020785808563, 'rewards/chosen': -4.1322407722473145, 'rewards/rejected': -4.378410816192627, 'logps/margins': 0.2909072935581207, 'logps/chosen': -9.470605850219727, 'logps/rejected': -9.761513710021973, 'accuracy': 0.5, 'epoch': 2.13}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 3860/5424 [1:07:45<20:39,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.9263, 'grad_norm': 4.122324466705322, 'learning_rate': 1.933334733105647e-06, 'rewards/margins': 0.4852231442928314, 'rewards/chosen': -5.846424579620361, 'rewards/rejected': -6.331647872924805, 'logps/margins': -0.8767927289009094, 'logps/chosen': -16.979328155517578, 'logps/rejected': -16.1025333404541, 'accuracy': 0.5, 'epoch': 2.13}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████▏  | 3870/5424 [1:07:52<20:33,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.4702, 'grad_norm': 5.596553802490234, 'learning_rate': 1.910512479931668e-06, 'rewards/margins': 2.1579487323760986, 'rewards/chosen': -5.004040718078613, 'rewards/rejected': -7.161989688873291, 'logps/margins': 10.769063949584961, 'logps/chosen': -9.962281227111816, 'logps/rejected': -20.731346130371094, 'accuracy': 0.699999988079071, 'epoch': 2.14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 3880/5424 [1:08:01<20:39,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.0168, 'grad_norm': 2.086592674255371, 'learning_rate': 1.8877938749253117e-06, 'rewards/margins': 0.9655635952949524, 'rewards/chosen': -4.909524440765381, 'rewards/rejected': -5.875087261199951, 'logps/margins': 4.582730293273926, 'logps/chosen': -11.899901390075684, 'logps/rejected': -16.48263168334961, 'accuracy': 0.800000011920929, 'epoch': 2.15}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 3890/5424 [1:08:09<22:45,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 25.4134, 'grad_norm': 6.235023021697998, 'learning_rate': 1.8651796802653449e-06, 'rewards/margins': 0.6051955223083496, 'rewards/chosen': -8.616143226623535, 'rewards/rejected': -9.221338272094727, 'logps/margins': 5.5128912925720215, 'logps/chosen': -23.04952621459961, 'logps/rejected': -28.56241798400879, 'accuracy': 0.6000000238418579, 'epoch': 2.15}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 3900/5424 [1:08:17<20:07,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.7731, 'grad_norm': 0.7521899342536926, 'learning_rate': 1.8426706546277001e-06, 'rewards/margins': 2.6549715995788574, 'rewards/chosen': -4.539296627044678, 'rewards/rejected': -7.194268226623535, 'logps/margins': 11.450439453125, 'logps/chosen': -9.951204299926758, 'logps/rejected': -21.40164566040039, 'accuracy': 0.800000011920929, 'epoch': 2.16}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 3910/5424 [1:08:25<20:03,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.4073, 'grad_norm': 2.112908124923706, 'learning_rate': 1.8202675531600362e-06, 'rewards/margins': 1.4029185771942139, 'rewards/chosen': -4.226494789123535, 'rewards/rejected': -5.629413604736328, 'logps/margins': 5.5221052169799805, 'logps/chosen': -9.564858436584473, 'logps/rejected': -15.086962699890137, 'accuracy': 0.6000000238418579, 'epoch': 2.16}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 3920/5424 [1:08:33<20:42,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.8472, 'grad_norm': 3.7371695041656494, 'learning_rate': 1.7979711274563966e-06, 'rewards/margins': 3.447211503982544, 'rewards/chosen': -5.246293067932129, 'rewards/rejected': -8.693504333496094, 'logps/margins': 16.50478744506836, 'logps/chosen': -11.659494400024414, 'logps/rejected': -28.16428565979004, 'accuracy': 0.8999999761581421, 'epoch': 2.17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 3930/5424 [1:08:42<20:20,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.2734, 'grad_norm': 3.9501495361328125, 'learning_rate': 1.775782125531997e-06, 'rewards/margins': 0.8581650853157043, 'rewards/chosen': -5.814471244812012, 'rewards/rejected': -6.67263650894165, 'logps/margins': 7.9257659912109375, 'logps/chosen': -13.988879203796387, 'logps/rejected': -21.91464614868164, 'accuracy': 0.6000000238418579, 'epoch': 2.17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 3940/5424 [1:08:50<19:45,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.6799, 'grad_norm': 1.9218283891677856, 'learning_rate': 1.753701291798124e-06, 'rewards/margins': 5.634952545166016, 'rewards/chosen': -5.597872734069824, 'rewards/rejected': -11.232824325561523, 'logps/margins': 26.545202255249023, 'logps/chosen': -12.563714981079102, 'logps/rejected': -39.10892105102539, 'accuracy': 1.0, 'epoch': 2.18}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 3950/5424 [1:08:58<19:49,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.8112, 'grad_norm': 4.136990070343018, 'learning_rate': 1.7317293670371733e-06, 'rewards/margins': 1.5543006658554077, 'rewards/chosen': -4.818201065063477, 'rewards/rejected': -6.372501850128174, 'logps/margins': 7.633211612701416, 'logps/chosen': -10.3473539352417, 'logps/rejected': -17.980566024780273, 'accuracy': 0.800000011920929, 'epoch': 2.18}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 3960/5424 [1:09:07<19:40,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.0934, 'grad_norm': 3.5924999713897705, 'learning_rate': 1.709867088377784e-06, 'rewards/margins': 2.428281307220459, 'rewards/chosen': -5.556559085845947, 'rewards/rejected': -7.98483943939209, 'logps/margins': 11.13847827911377, 'logps/chosen': -12.599637031555176, 'logps/rejected': -23.738115310668945, 'accuracy': 0.699999988079071, 'epoch': 2.19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 3970/5424 [1:09:15<19:31,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.2129, 'grad_norm': 2.8239314556121826, 'learning_rate': 1.6881151892701215e-06, 'rewards/margins': 3.401780605316162, 'rewards/chosen': -4.7390360832214355, 'rewards/rejected': -8.140817642211914, 'logps/margins': 16.994272232055664, 'logps/chosen': -11.676186561584473, 'logps/rejected': -28.670459747314453, 'accuracy': 0.699999988079071, 'epoch': 2.2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 3980/5424 [1:09:23<19:16,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 21.3625, 'grad_norm': 3.334186553955078, 'learning_rate': 1.6664743994612586e-06, 'rewards/margins': 0.0009014129755087197, 'rewards/chosen': -6.480027675628662, 'rewards/rejected': -6.480928897857666, 'logps/margins': 0.6261776685714722, 'logps/chosen': -18.937419891357422, 'logps/rejected': -19.5635986328125, 'accuracy': 0.5, 'epoch': 2.2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▎  | 3990/5424 [1:09:31<19:37,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.4664, 'grad_norm': 8.172540664672852, 'learning_rate': 1.644945444970706e-06, 'rewards/margins': 2.0838332176208496, 'rewards/chosen': -5.864614486694336, 'rewards/rejected': -7.948447227478027, 'logps/margins': 10.623979568481445, 'logps/chosen': -15.11645221710205, 'logps/rejected': -25.740427017211914, 'accuracy': 0.6000000238418579, 'epoch': 2.21}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▎  | 4000/5424 [1:09:39<19:58,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.1588, 'grad_norm': 3.6758947372436523, 'learning_rate': 1.6235290480660448e-06, 'rewards/margins': 2.2192330360412598, 'rewards/chosen': -5.299873352050781, 'rewards/rejected': -7.519105434417725, 'logps/margins': 13.189831733703613, 'logps/chosen': -12.209437370300293, 'logps/rejected': -25.399269104003906, 'accuracy': 0.5, 'epoch': 2.21}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▍  | 4010/5424 [1:09:48<19:12,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.822, 'grad_norm': 2.8756120204925537, 'learning_rate': 1.6022259272387036e-06, 'rewards/margins': 1.8988125324249268, 'rewards/chosen': -4.990065574645996, 'rewards/rejected': -6.888877868652344, 'logps/margins': 8.737353324890137, 'logps/chosen': -12.761120796203613, 'logps/rejected': -21.498470306396484, 'accuracy': 0.6000000238418579, 'epoch': 2.22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▍  | 4020/5424 [1:09:56<19:58,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.9439, 'grad_norm': 4.106810092926025, 'learning_rate': 1.5810367971798496e-06, 'rewards/margins': 1.520948886871338, 'rewards/chosen': -5.71983003616333, 'rewards/rejected': -7.240778923034668, 'logps/margins': 7.65084981918335, 'logps/chosen': -14.363424301147461, 'logps/rejected': -22.0142765045166, 'accuracy': 0.699999988079071, 'epoch': 2.22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▍  | 4030/5424 [1:10:04<19:26,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.8167, 'grad_norm': 3.646176338195801, 'learning_rate': 1.5599623687564076e-06, 'rewards/margins': 3.1683154106140137, 'rewards/chosen': -5.781529426574707, 'rewards/rejected': -8.949845314025879, 'logps/margins': 16.450275421142578, 'logps/chosen': -12.300661087036133, 'logps/rejected': -28.750940322875977, 'accuracy': 0.699999988079071, 'epoch': 2.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▍  | 4040/5424 [1:10:12<18:27,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.4943, 'grad_norm': 4.046566486358643, 'learning_rate': 1.5390033489872214e-06, 'rewards/margins': 3.0471432209014893, 'rewards/chosen': -4.918219089508057, 'rewards/rejected': -7.965361595153809, 'logps/margins': 14.905278205871582, 'logps/chosen': -10.949551582336426, 'logps/rejected': -25.85483169555664, 'accuracy': 0.699999988079071, 'epoch': 2.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▍  | 4050/5424 [1:10:20<19:29,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.0971, 'grad_norm': 4.637202262878418, 'learning_rate': 1.51816044101933e-06, 'rewards/margins': 0.2521868348121643, 'rewards/chosen': -5.300505638122559, 'rewards/rejected': -5.552692413330078, 'logps/margins': -0.4940221905708313, 'logps/chosen': -14.1187105178833, 'logps/rejected': -13.624687194824219, 'accuracy': 0.6000000238418579, 'epoch': 2.24}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▍  | 4060/5424 [1:10:29<18:08,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.094, 'grad_norm': 3.8285439014434814, 'learning_rate': 1.4974343441043715e-06, 'rewards/margins': 2.9244465827941895, 'rewards/chosen': -4.7608771324157715, 'rewards/rejected': -7.685323238372803, 'logps/margins': 12.653059005737305, 'logps/chosen': -10.694536209106445, 'logps/rejected': -23.34759521484375, 'accuracy': 0.800000011920929, 'epoch': 2.25}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 4068/5424 [1:10:35<18:18,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating response...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/201 [00:00<?, ?it/s]\u001b[A\n",
      "  1%|          | 2/201 [00:00<01:05,  3.03it/s]\u001b[A\n",
      "  1%|▏         | 3/201 [00:01<01:42,  1.93it/s]\u001b[A\n",
      "  2%|▏         | 4/201 [00:02<01:48,  1.82it/s]\u001b[A\n",
      "  2%|▏         | 5/201 [00:02<01:54,  1.72it/s]\u001b[A\n",
      "  3%|▎         | 6/201 [00:03<01:58,  1.64it/s]\u001b[A\n",
      "  3%|▎         | 7/201 [00:04<02:01,  1.60it/s]\u001b[A\n",
      "  4%|▍         | 8/201 [00:04<02:03,  1.56it/s]\u001b[A\n",
      "  4%|▍         | 9/201 [00:05<02:11,  1.46it/s]\u001b[A\n",
      "  5%|▍         | 10/201 [00:06<02:05,  1.52it/s]\u001b[A\n",
      "  5%|▌         | 11/201 [00:06<02:05,  1.52it/s]\u001b[A\n",
      "  6%|▌         | 12/201 [00:07<02:05,  1.51it/s]\u001b[A\n",
      "  6%|▋         | 13/201 [00:08<02:10,  1.44it/s]\u001b[A\n",
      "  7%|▋         | 14/201 [00:08<02:07,  1.47it/s]\u001b[A\n",
      "  7%|▋         | 15/201 [00:09<02:06,  1.47it/s]\u001b[A\n",
      "  8%|▊         | 16/201 [00:10<02:05,  1.48it/s]\u001b[A\n",
      "  8%|▊         | 17/201 [00:10<02:09,  1.42it/s]\u001b[A\n",
      "  9%|▉         | 18/201 [00:11<02:08,  1.42it/s]\u001b[A\n",
      "  9%|▉         | 19/201 [00:12<02:08,  1.42it/s]\u001b[A\n",
      " 10%|▉         | 20/201 [00:13<02:05,  1.45it/s]\u001b[A\n",
      " 10%|█         | 21/201 [00:13<02:08,  1.40it/s]\u001b[A\n",
      " 11%|█         | 22/201 [00:14<02:07,  1.41it/s]\u001b[A\n",
      " 11%|█▏        | 23/201 [00:15<02:11,  1.36it/s]\u001b[A\n",
      " 12%|█▏        | 24/201 [00:15<02:05,  1.41it/s]\u001b[A\n",
      " 12%|█▏        | 25/201 [00:16<02:03,  1.43it/s]\u001b[A\n",
      " 13%|█▎        | 26/201 [00:17<02:00,  1.46it/s]\u001b[A\n",
      " 13%|█▎        | 27/201 [00:17<01:58,  1.47it/s]\u001b[A\n",
      " 14%|█▍        | 28/201 [00:18<02:02,  1.41it/s]\u001b[A\n",
      " 14%|█▍        | 29/201 [00:19<02:00,  1.43it/s]\u001b[A\n",
      " 15%|█▍        | 30/201 [00:20<01:58,  1.45it/s]\u001b[A\n",
      " 15%|█▌        | 31/201 [00:20<01:55,  1.48it/s]\u001b[A\n",
      " 16%|█▌        | 32/201 [00:21<01:52,  1.50it/s]\u001b[A\n",
      " 16%|█▋        | 33/201 [00:22<01:57,  1.43it/s]\u001b[A\n",
      " 17%|█▋        | 34/201 [00:22<01:55,  1.45it/s]\u001b[A\n",
      " 17%|█▋        | 35/201 [00:23<01:52,  1.47it/s]\u001b[A\n",
      " 18%|█▊        | 36/201 [00:24<01:51,  1.47it/s]\u001b[A\n",
      " 18%|█▊        | 37/201 [00:24<01:51,  1.48it/s]\u001b[A\n",
      " 19%|█▉        | 38/201 [00:25<01:48,  1.50it/s]\u001b[A\n",
      " 19%|█▉        | 39/201 [00:26<01:48,  1.49it/s]\u001b[A\n",
      " 20%|█▉        | 40/201 [00:26<01:47,  1.50it/s]\u001b[A\n",
      " 20%|██        | 41/201 [00:27<01:43,  1.54it/s]\u001b[A\n",
      " 21%|██        | 42/201 [00:28<01:49,  1.46it/s]\u001b[A\n",
      " 21%|██▏       | 43/201 [00:28<01:46,  1.48it/s]\u001b[A\n",
      " 22%|██▏       | 44/201 [00:29<01:45,  1.49it/s]\u001b[A\n",
      " 22%|██▏       | 45/201 [00:30<01:48,  1.43it/s]\u001b[A\n",
      " 23%|██▎       | 46/201 [00:30<01:46,  1.45it/s]\u001b[A\n",
      " 23%|██▎       | 47/201 [00:31<01:45,  1.47it/s]\u001b[A\n",
      " 24%|██▍       | 48/201 [00:32<01:40,  1.52it/s]\u001b[A\n",
      " 24%|██▍       | 49/201 [00:32<01:37,  1.56it/s]\u001b[A\n",
      " 25%|██▍       | 50/201 [00:33<01:37,  1.54it/s]\u001b[A\n",
      " 25%|██▌       | 51/201 [00:34<01:37,  1.54it/s]\u001b[A\n",
      " 26%|██▌       | 52/201 [00:34<01:37,  1.53it/s]\u001b[A\n",
      " 26%|██▋       | 53/201 [00:35<01:37,  1.52it/s]\u001b[A\n",
      " 27%|██▋       | 54/201 [00:36<01:41,  1.45it/s]\u001b[A\n",
      " 27%|██▋       | 55/201 [00:36<01:39,  1.47it/s]\u001b[A\n",
      " 28%|██▊       | 56/201 [00:37<01:37,  1.48it/s]\u001b[A\n",
      " 28%|██▊       | 57/201 [00:38<01:41,  1.41it/s]\u001b[A\n",
      " 29%|██▉       | 58/201 [00:38<01:40,  1.43it/s]\u001b[A\n",
      " 29%|██▉       | 59/201 [00:39<01:37,  1.46it/s]\u001b[A\n",
      " 30%|██▉       | 60/201 [00:40<01:35,  1.48it/s]\u001b[A\n",
      " 30%|███       | 61/201 [00:40<01:34,  1.49it/s]\u001b[A\n",
      " 31%|███       | 62/201 [00:41<01:33,  1.49it/s]\u001b[A\n",
      " 31%|███▏      | 63/201 [00:42<01:33,  1.48it/s]\u001b[A\n",
      " 32%|███▏      | 64/201 [00:42<01:32,  1.49it/s]\u001b[A\n",
      " 32%|███▏      | 65/201 [00:43<01:31,  1.48it/s]\u001b[A\n",
      " 33%|███▎      | 66/201 [00:44<01:35,  1.42it/s]\u001b[A\n",
      " 33%|███▎      | 67/201 [00:45<01:33,  1.43it/s]\u001b[A\n",
      " 34%|███▍      | 68/201 [00:45<01:31,  1.45it/s]\u001b[A\n",
      " 34%|███▍      | 69/201 [00:46<01:27,  1.51it/s]\u001b[A\n",
      " 35%|███▍      | 70/201 [00:47<01:26,  1.51it/s]\u001b[A\n",
      " 35%|███▌      | 71/201 [00:47<01:26,  1.50it/s]\u001b[A\n",
      " 36%|███▌      | 72/201 [00:48<01:25,  1.51it/s]\u001b[A\n",
      " 36%|███▋      | 73/201 [00:49<01:25,  1.50it/s]\u001b[A\n",
      " 37%|███▋      | 74/201 [00:49<01:23,  1.51it/s]\u001b[A\n",
      " 37%|███▋      | 75/201 [00:50<01:27,  1.43it/s]\u001b[A\n",
      " 38%|███▊      | 76/201 [00:51<01:26,  1.45it/s]\u001b[A\n",
      " 38%|███▊      | 77/201 [00:51<01:24,  1.47it/s]\u001b[A\n",
      " 39%|███▉      | 78/201 [00:52<01:27,  1.41it/s]\u001b[A\n",
      " 39%|███▉      | 79/201 [00:53<01:29,  1.37it/s]\u001b[A\n",
      " 40%|███▉      | 80/201 [00:54<01:29,  1.35it/s]\u001b[A\n",
      " 40%|████      | 81/201 [00:54<01:25,  1.40it/s]\u001b[A\n",
      " 41%|████      | 82/201 [00:55<01:27,  1.36it/s]\u001b[A\n",
      " 41%|████▏     | 83/201 [00:56<01:28,  1.34it/s]\u001b[A\n",
      " 42%|████▏     | 84/201 [00:56<01:23,  1.39it/s]\u001b[A\n",
      " 42%|████▏     | 85/201 [00:57<01:20,  1.43it/s]\u001b[A\n",
      " 43%|████▎     | 86/201 [00:58<01:18,  1.46it/s]\u001b[A\n",
      " 43%|████▎     | 87/201 [00:58<01:16,  1.49it/s]\u001b[A\n",
      " 44%|████▍     | 88/201 [00:59<01:15,  1.50it/s]\u001b[A\n",
      " 44%|████▍     | 89/201 [01:00<01:18,  1.42it/s]\u001b[A\n",
      " 45%|████▍     | 90/201 [01:01<01:16,  1.46it/s]\u001b[A\n",
      " 45%|████▌     | 91/201 [01:01<01:12,  1.51it/s]\u001b[A\n",
      " 46%|████▌     | 92/201 [01:02<01:15,  1.44it/s]\u001b[A\n",
      " 46%|████▋     | 93/201 [01:03<01:14,  1.46it/s]\u001b[A\n",
      " 47%|████▋     | 94/201 [01:03<01:12,  1.47it/s]\u001b[A\n",
      " 47%|████▋     | 95/201 [01:04<01:11,  1.48it/s]\u001b[A\n",
      " 48%|████▊     | 96/201 [01:04<01:08,  1.53it/s]\u001b[A\n",
      " 48%|████▊     | 97/201 [01:05<01:08,  1.52it/s]\u001b[A\n",
      " 49%|████▉     | 98/201 [01:06<01:07,  1.53it/s]\u001b[A\n",
      " 49%|████▉     | 99/201 [01:06<01:07,  1.52it/s]\u001b[A\n",
      " 50%|████▉     | 100/201 [01:07<01:06,  1.52it/s]\u001b[A\n",
      " 50%|█████     | 101/201 [01:08<01:05,  1.52it/s]\u001b[A\n",
      " 51%|█████     | 102/201 [01:08<01:04,  1.52it/s]\u001b[A\n",
      " 51%|█████     | 103/201 [01:09<01:04,  1.51it/s]\u001b[A\n",
      " 52%|█████▏    | 104/201 [01:10<01:05,  1.49it/s]\u001b[A\n",
      " 52%|█████▏    | 105/201 [01:10<01:04,  1.48it/s]\u001b[A\n",
      " 53%|█████▎    | 106/201 [01:11<01:03,  1.50it/s]\u001b[A\n",
      " 53%|█████▎    | 107/201 [01:12<01:02,  1.51it/s]\u001b[A\n",
      " 54%|█████▎    | 108/201 [01:12<01:01,  1.52it/s]\u001b[A\n",
      " 54%|█████▍    | 109/201 [01:13<01:00,  1.53it/s]\u001b[A\n",
      " 55%|█████▍    | 110/201 [01:14<00:58,  1.56it/s]\u001b[A\n",
      " 55%|█████▌    | 111/201 [01:14<00:58,  1.54it/s]\u001b[A\n",
      " 56%|█████▌    | 112/201 [01:15<00:58,  1.52it/s]\u001b[A\n",
      " 56%|█████▌    | 113/201 [01:16<00:57,  1.53it/s]\u001b[A\n",
      " 57%|█████▋    | 114/201 [01:16<00:56,  1.53it/s]\u001b[A\n",
      " 57%|█████▋    | 115/201 [01:17<00:57,  1.50it/s]\u001b[A\n",
      " 58%|█████▊    | 116/201 [01:18<00:56,  1.51it/s]\u001b[A\n",
      " 58%|█████▊    | 117/201 [01:18<00:55,  1.51it/s]\u001b[A\n",
      " 59%|█████▊    | 118/201 [01:19<00:54,  1.53it/s]\u001b[A\n",
      " 59%|█████▉    | 119/201 [01:20<00:53,  1.53it/s]\u001b[A\n",
      " 60%|█████▉    | 120/201 [01:20<00:53,  1.52it/s]\u001b[A\n",
      " 60%|██████    | 121/201 [01:21<00:52,  1.53it/s]\u001b[A\n",
      " 61%|██████    | 122/201 [01:22<00:52,  1.51it/s]\u001b[A\n",
      " 61%|██████    | 123/201 [01:22<00:53,  1.45it/s]\u001b[A\n",
      " 62%|██████▏   | 124/201 [01:23<00:52,  1.46it/s]\u001b[A\n",
      " 62%|██████▏   | 125/201 [01:24<00:50,  1.52it/s]\u001b[A\n",
      " 63%|██████▎   | 126/201 [01:24<00:48,  1.56it/s]\u001b[A\n",
      " 63%|██████▎   | 127/201 [01:25<00:47,  1.55it/s]\u001b[A\n",
      " 64%|██████▎   | 128/201 [01:26<00:47,  1.55it/s]\u001b[A\n",
      " 64%|██████▍   | 129/201 [01:26<00:46,  1.55it/s]\u001b[A\n",
      " 65%|██████▍   | 130/201 [01:27<00:46,  1.54it/s]\u001b[A\n",
      " 65%|██████▌   | 131/201 [01:27<00:45,  1.54it/s]\u001b[A\n",
      " 66%|██████▌   | 132/201 [01:28<00:49,  1.40it/s]\u001b[A\n",
      " 66%|██████▌   | 133/201 [01:29<00:48,  1.41it/s]\u001b[A\n",
      " 67%|██████▋   | 134/201 [01:30<00:46,  1.44it/s]\u001b[A\n",
      " 67%|██████▋   | 135/201 [01:30<00:44,  1.47it/s]\u001b[A\n",
      " 68%|██████▊   | 136/201 [01:31<00:43,  1.49it/s]\u001b[A\n",
      " 68%|██████▊   | 137/201 [01:32<00:43,  1.49it/s]\u001b[A\n",
      " 69%|██████▊   | 138/201 [01:32<00:42,  1.49it/s]\u001b[A\n",
      " 69%|██████▉   | 139/201 [01:33<00:41,  1.51it/s]\u001b[A\n",
      " 70%|██████▉   | 140/201 [01:34<00:40,  1.51it/s]\u001b[A\n",
      " 70%|███████   | 141/201 [01:34<00:39,  1.51it/s]\u001b[A\n",
      " 71%|███████   | 142/201 [01:35<00:41,  1.44it/s]\u001b[A\n",
      " 71%|███████   | 143/201 [01:36<00:39,  1.45it/s]\u001b[A\n",
      " 72%|███████▏  | 144/201 [01:36<00:38,  1.46it/s]\u001b[A\n",
      " 72%|███████▏  | 145/201 [01:37<00:37,  1.49it/s]\u001b[A\n",
      " 73%|███████▎  | 146/201 [01:38<00:38,  1.42it/s]\u001b[A\n",
      " 73%|███████▎  | 147/201 [01:39<00:39,  1.37it/s]\u001b[A\n",
      " 74%|███████▎  | 148/201 [01:39<00:37,  1.42it/s]\u001b[A\n",
      " 74%|███████▍  | 149/201 [01:40<00:36,  1.44it/s]\u001b[A\n",
      " 75%|███████▍  | 150/201 [01:41<00:35,  1.46it/s]\u001b[A\n",
      " 75%|███████▌  | 151/201 [01:41<00:35,  1.39it/s]\u001b[A\n",
      " 76%|███████▌  | 152/201 [01:42<00:34,  1.42it/s]\u001b[A\n",
      " 76%|███████▌  | 153/201 [01:43<00:33,  1.44it/s]\u001b[A\n",
      " 77%|███████▋  | 154/201 [01:43<00:31,  1.47it/s]\u001b[A\n",
      " 77%|███████▋  | 155/201 [01:44<00:31,  1.47it/s]\u001b[A\n",
      " 78%|███████▊  | 156/201 [01:45<00:30,  1.49it/s]\u001b[A\n",
      " 78%|███████▊  | 157/201 [01:45<00:29,  1.47it/s]\u001b[A\n",
      " 79%|███████▊  | 158/201 [01:46<00:28,  1.49it/s]\u001b[A\n",
      " 79%|███████▉  | 159/201 [01:47<00:27,  1.51it/s]\u001b[A\n",
      " 80%|███████▉  | 160/201 [01:47<00:27,  1.52it/s]\u001b[A\n",
      " 80%|████████  | 161/201 [01:48<00:26,  1.53it/s]\u001b[A\n",
      " 81%|████████  | 162/201 [01:49<00:25,  1.53it/s]\u001b[A\n",
      " 81%|████████  | 163/201 [01:49<00:24,  1.52it/s]\u001b[A\n",
      " 82%|████████▏ | 164/201 [01:50<00:24,  1.52it/s]\u001b[A\n",
      " 82%|████████▏ | 165/201 [01:51<00:24,  1.49it/s]\u001b[A\n",
      " 83%|████████▎ | 166/201 [01:51<00:23,  1.49it/s]\u001b[A\n",
      " 83%|████████▎ | 167/201 [01:52<00:22,  1.49it/s]\u001b[A\n",
      " 84%|████████▎ | 168/201 [01:53<00:23,  1.40it/s]\u001b[A\n",
      " 84%|████████▍ | 169/201 [01:54<00:22,  1.43it/s]\u001b[A\n",
      " 85%|████████▍ | 170/201 [01:54<00:21,  1.44it/s]\u001b[A\n",
      " 85%|████████▌ | 171/201 [01:55<00:20,  1.47it/s]\u001b[A\n",
      " 86%|████████▌ | 172/201 [01:56<00:20,  1.41it/s]\u001b[A\n",
      " 86%|████████▌ | 173/201 [01:56<00:19,  1.43it/s]\u001b[A\n",
      " 87%|████████▋ | 174/201 [01:57<00:18,  1.45it/s]\u001b[A\n",
      " 87%|████████▋ | 175/201 [01:58<00:17,  1.46it/s]\u001b[A\n",
      " 88%|████████▊ | 176/201 [01:58<00:16,  1.48it/s]\u001b[A\n",
      " 88%|████████▊ | 177/201 [01:59<00:16,  1.49it/s]\u001b[A\n",
      " 89%|████████▊ | 178/201 [02:00<00:15,  1.49it/s]\u001b[A\n",
      " 89%|████████▉ | 179/201 [02:00<00:14,  1.51it/s]\u001b[A\n",
      " 90%|████████▉ | 180/201 [02:01<00:13,  1.51it/s]\u001b[A\n",
      " 90%|█████████ | 181/201 [02:02<00:13,  1.52it/s]\u001b[A\n",
      " 91%|█████████ | 182/201 [02:02<00:12,  1.52it/s]\u001b[A\n",
      " 91%|█████████ | 183/201 [02:03<00:11,  1.53it/s]\u001b[A\n",
      " 92%|█████████▏| 184/201 [02:04<00:11,  1.51it/s]\u001b[A\n",
      " 92%|█████████▏| 185/201 [02:04<00:10,  1.52it/s]\u001b[A\n",
      " 93%|█████████▎| 186/201 [02:05<00:09,  1.51it/s]\u001b[A\n",
      " 93%|█████████▎| 187/201 [02:06<00:09,  1.50it/s]\u001b[A\n",
      " 94%|█████████▎| 188/201 [02:06<00:08,  1.50it/s]\u001b[A\n",
      " 94%|█████████▍| 189/201 [02:07<00:07,  1.50it/s]\u001b[A\n",
      " 95%|█████████▍| 190/201 [02:08<00:07,  1.50it/s]\u001b[A\n",
      " 95%|█████████▌| 191/201 [02:08<00:06,  1.51it/s]\u001b[A\n",
      " 96%|█████████▌| 192/201 [02:09<00:05,  1.52it/s]\u001b[A\n",
      " 96%|█████████▌| 193/201 [02:10<00:05,  1.51it/s]\u001b[A\n",
      " 97%|█████████▋| 194/201 [02:10<00:04,  1.55it/s]\u001b[A\n",
      " 97%|█████████▋| 195/201 [02:11<00:03,  1.55it/s]\u001b[A\n",
      " 98%|█████████▊| 196/201 [02:11<00:03,  1.53it/s]\u001b[A\n",
      " 98%|█████████▊| 197/201 [02:12<00:02,  1.52it/s]\u001b[A\n",
      " 99%|█████████▊| 198/201 [02:13<00:01,  1.53it/s]\u001b[A\n",
      " 99%|█████████▉| 199/201 [02:13<00:01,  1.53it/s]\u001b[A\n",
      "100%|█████████▉| 200/201 [02:14<00:00,  1.45it/s]\u001b[A\n",
      "                                                     \n",
      " 75%|███████▌  | 4068/5424 [1:13:03<18:18,  1.23it/s]\n",
      "100%|██████████| 201/201 [02:15<00:00,  1.48it/s]\u001b[A\n",
      "                                                 \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 14.57752513885498, 'eval_accuracy': 0.5970149253731343, 'eval_runtime': 147.8934, 'eval_samples_per_second': 1.359, 'eval_steps_per_second': 1.359, 'eval_rewards/margins': 1.292073369026184, 'eval_rewards/chosen': -5.493359565734863, 'eval_rewards/rejected': -6.785433292388916, 'eval_logps/margins': 7.8208465576171875, 'eval_logps/chosen': -13.323068618774414, 'eval_logps/rejected': -21.14391326904297, 'epoch': 2.25}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 4070/5424 [1:13:14<12:45:16, 33.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.9648, 'grad_norm': 3.4139926433563232, 'learning_rate': 1.4768257535751347e-06, 'rewards/margins': 1.3839538097381592, 'rewards/chosen': -5.3709845542907715, 'rewards/rejected': -6.754939079284668, 'logps/margins': 9.86713981628418, 'logps/chosen': -11.856995582580566, 'logps/rejected': -21.72413444519043, 'accuracy': 0.699999988079071, 'epoch': 2.25}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 4080/5424 [1:13:23<40:23,  1.80s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 21.1454, 'grad_norm': 4.050633907318115, 'learning_rate': 1.4563353608222226e-06, 'rewards/margins': 0.8691112399101257, 'rewards/chosen': -7.017066955566406, 'rewards/rejected': -7.886178016662598, 'logps/margins': 6.133049488067627, 'logps/chosen': -19.042957305908203, 'logps/rejected': -25.17600440979004, 'accuracy': 0.5, 'epoch': 2.26}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 4090/5424 [1:13:31<18:42,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.2287, 'grad_norm': 4.34482479095459, 'learning_rate': 1.4359638532708641e-06, 'rewards/margins': 2.9660019874572754, 'rewards/chosen': -4.680461883544922, 'rewards/rejected': -7.646463871002197, 'logps/margins': 15.798585891723633, 'logps/chosen': -8.886524200439453, 'logps/rejected': -24.685110092163086, 'accuracy': 0.800000011920929, 'epoch': 2.26}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 4100/5424 [1:13:39<17:28,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.9994, 'grad_norm': 3.5183305740356445, 'learning_rate': 1.4157119143578497e-06, 'rewards/margins': 2.17218017578125, 'rewards/chosen': -4.880921363830566, 'rewards/rejected': -7.053101539611816, 'logps/margins': 9.445408821105957, 'logps/chosen': -10.909560203552246, 'logps/rejected': -20.354969024658203, 'accuracy': 0.6000000238418579, 'epoch': 2.27}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 4110/5424 [1:13:47<18:27,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.2376, 'grad_norm': 11.545798301696777, 'learning_rate': 1.3955802235085986e-06, 'rewards/margins': 1.3416264057159424, 'rewards/chosen': -5.958540439605713, 'rewards/rejected': -7.300167083740234, 'logps/margins': 8.168156623840332, 'logps/chosen': -14.811410903930664, 'logps/rejected': -22.979564666748047, 'accuracy': 0.4000000059604645, 'epoch': 2.27}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 4120/5424 [1:13:56<17:23,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.2603, 'grad_norm': 3.5145890712738037, 'learning_rate': 1.3755694561143689e-06, 'rewards/margins': 1.4207184314727783, 'rewards/chosen': -5.67910099029541, 'rewards/rejected': -7.099820137023926, 'logps/margins': 7.155647277832031, 'logps/chosen': -14.050191879272461, 'logps/rejected': -21.205839157104492, 'accuracy': 0.5, 'epoch': 2.28}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 4130/5424 [1:14:04<17:42,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.3865, 'grad_norm': 2.3683900833129883, 'learning_rate': 1.3556802835096023e-06, 'rewards/margins': 1.27290940284729, 'rewards/chosen': -5.030817985534668, 'rewards/rejected': -6.303727149963379, 'logps/margins': 4.22153377532959, 'logps/chosen': -13.9614896774292, 'logps/rejected': -18.18302345275879, 'accuracy': 0.800000011920929, 'epoch': 2.28}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▋  | 4140/5424 [1:14:12<17:47,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.5221, 'grad_norm': 6.667713165283203, 'learning_rate': 1.3359133729493962e-06, 'rewards/margins': 1.597856044769287, 'rewards/chosen': -5.5101776123046875, 'rewards/rejected': -7.108033180236816, 'logps/margins': 6.71088171005249, 'logps/chosen': -13.61688232421875, 'logps/rejected': -20.327762603759766, 'accuracy': 0.6000000238418579, 'epoch': 2.29}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 4150/5424 [1:14:20<16:58,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.5072, 'grad_norm': 5.838793754577637, 'learning_rate': 1.3162693875871207e-06, 'rewards/margins': 0.28904905915260315, 'rewards/chosen': -5.920292377471924, 'rewards/rejected': -6.209342002868652, 'logps/margins': 5.3059797286987305, 'logps/chosen': -14.348920822143555, 'logps/rejected': -19.6549015045166, 'accuracy': 0.4000000059604645, 'epoch': 2.3}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 4160/5424 [1:14:28<17:04,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.8181, 'grad_norm': 4.267144680023193, 'learning_rate': 1.2967489864521676e-06, 'rewards/margins': 2.0962157249450684, 'rewards/chosen': -5.419489860534668, 'rewards/rejected': -7.5157060623168945, 'logps/margins': 11.448963165283203, 'logps/chosen': -12.536581039428711, 'logps/rejected': -23.98554229736328, 'accuracy': 0.5, 'epoch': 2.3}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 4170/5424 [1:14:36<16:41,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.1888, 'grad_norm': 7.133947372436523, 'learning_rate': 1.277352824427846e-06, 'rewards/margins': 0.25469306111335754, 'rewards/chosen': -5.757489204406738, 'rewards/rejected': -6.012182712554932, 'logps/margins': 3.112433671951294, 'logps/chosen': -15.501238822937012, 'logps/rejected': -18.61367416381836, 'accuracy': 0.5, 'epoch': 2.31}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 4180/5424 [1:14:45<16:59,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.3571, 'grad_norm': 2.094714641571045, 'learning_rate': 1.2580815522294115e-06, 'rewards/margins': 0.6630595326423645, 'rewards/chosen': -5.671648979187012, 'rewards/rejected': -6.334708213806152, 'logps/margins': 4.050869941711426, 'logps/chosen': -15.684843063354492, 'logps/rejected': -19.735713958740234, 'accuracy': 0.6000000238418579, 'epoch': 2.31}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 4190/5424 [1:14:53<17:14,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.5002, 'grad_norm': 4.129371643066406, 'learning_rate': 1.2389358163822268e-06, 'rewards/margins': 0.655268669128418, 'rewards/chosen': -5.129880428314209, 'rewards/rejected': -5.785149097442627, 'logps/margins': 4.317929744720459, 'logps/chosen': -12.96088695526123, 'logps/rejected': -17.2788143157959, 'accuracy': 0.5, 'epoch': 2.32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 4200/5424 [1:15:01<17:02,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.1445, 'grad_norm': 4.051896095275879, 'learning_rate': 1.2199162592000846e-06, 'rewards/margins': 0.11978320777416229, 'rewards/chosen': -6.6206231117248535, 'rewards/rejected': -6.740406036376953, 'logps/margins': 2.423682689666748, 'logps/chosen': -18.325313568115234, 'logps/rejected': -20.74899673461914, 'accuracy': 0.4000000059604645, 'epoch': 2.32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 4210/5424 [1:15:09<16:15,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.9643, 'grad_norm': 7.943828582763672, 'learning_rate': 1.2010235187636476e-06, 'rewards/margins': 0.3881092071533203, 'rewards/chosen': -4.532365322113037, 'rewards/rejected': -4.920474052429199, 'logps/margins': 4.9827470779418945, 'logps/chosen': -9.847623825073242, 'logps/rejected': -14.83036994934082, 'accuracy': 0.699999988079071, 'epoch': 2.33}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 4220/5424 [1:15:17<16:30,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.2806, 'grad_norm': 7.245777130126953, 'learning_rate': 1.1822582288990503e-06, 'rewards/margins': 1.4975265264511108, 'rewards/chosen': -4.9061479568481445, 'rewards/rejected': -6.4036736488342285, 'logps/margins': 8.222219467163086, 'logps/chosen': -11.312804222106934, 'logps/rejected': -19.535024642944336, 'accuracy': 0.6000000238418579, 'epoch': 2.33}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 4230/5424 [1:15:26<16:25,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 24.2132, 'grad_norm': 7.532875061035156, 'learning_rate': 1.1636210191566283e-06, 'rewards/margins': -0.0854012742638588, 'rewards/chosen': -7.0438947677612305, 'rewards/rejected': -6.958492279052734, 'logps/margins': -0.9247499704360962, 'logps/chosen': -22.192874908447266, 'logps/rejected': -21.268125534057617, 'accuracy': 0.6000000238418579, 'epoch': 2.34}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 4240/5424 [1:15:34<16:24,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.9984, 'grad_norm': 3.611363410949707, 'learning_rate': 1.1451125147898052e-06, 'rewards/margins': 2.519557476043701, 'rewards/chosen': -4.610805988311768, 'rewards/rejected': -7.130363464355469, 'logps/margins': 10.153417587280273, 'logps/chosen': -11.063444137573242, 'logps/rejected': -21.216861724853516, 'accuracy': 0.6000000238418579, 'epoch': 2.35}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 4250/5424 [1:15:42<15:50,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.1817, 'grad_norm': 5.142885208129883, 'learning_rate': 1.126733336734105e-06, 'rewards/margins': 0.751297116279602, 'rewards/chosen': -6.339760780334473, 'rewards/rejected': -7.091057777404785, 'logps/margins': 6.978428840637207, 'logps/chosen': -16.345134735107422, 'logps/rejected': -23.323562622070312, 'accuracy': 0.6000000238418579, 'epoch': 2.35}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▊  | 4260/5424 [1:15:50<15:38,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 21.4279, 'grad_norm': 3.6259028911590576, 'learning_rate': 1.1084841015863318e-06, 'rewards/margins': -0.7572464942932129, 'rewards/chosen': -7.156097412109375, 'rewards/rejected': -6.39885139465332, 'logps/margins': 0.7556358575820923, 'logps/chosen': -19.053592681884766, 'logps/rejected': -19.80923080444336, 'accuracy': 0.699999988079071, 'epoch': 2.36}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▊  | 4270/5424 [1:15:58<15:07,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.2674, 'grad_norm': 2.1159396171569824, 'learning_rate': 1.0903654215838782e-06, 'rewards/margins': 1.3610734939575195, 'rewards/chosen': -4.3676605224609375, 'rewards/rejected': -5.728734016418457, 'logps/margins': 5.488409996032715, 'logps/chosen': -8.716290473937988, 'logps/rejected': -14.204699516296387, 'accuracy': 0.6000000238418579, 'epoch': 2.36}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▉  | 4280/5424 [1:16:06<15:20,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.0697, 'grad_norm': 4.997529983520508, 'learning_rate': 1.072377904584188e-06, 'rewards/margins': 1.1515347957611084, 'rewards/chosen': -5.0693359375, 'rewards/rejected': -6.220870018005371, 'logps/margins': 9.124260902404785, 'logps/chosen': -11.266605377197266, 'logps/rejected': -20.390865325927734, 'accuracy': 0.699999988079071, 'epoch': 2.37}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▉  | 4290/5424 [1:16:14<15:17,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.2203, 'grad_norm': 3.038085699081421, 'learning_rate': 1.05452215404436e-06, 'rewards/margins': 0.3915421664714813, 'rewards/chosen': -4.925750255584717, 'rewards/rejected': -5.3172926902771, 'logps/margins': 5.530745506286621, 'logps/chosen': -9.392093658447266, 'logps/rejected': -14.922839164733887, 'accuracy': 0.5, 'epoch': 2.37}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▉  | 4300/5424 [1:16:23<15:05,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.4194, 'grad_norm': 1.038214087486267, 'learning_rate': 1.036798769000903e-06, 'rewards/margins': 2.107421875, 'rewards/chosen': -4.9173688888549805, 'rewards/rejected': -7.0247907638549805, 'logps/margins': 12.324649810791016, 'logps/chosen': -12.399291038513184, 'logps/rejected': -24.723941802978516, 'accuracy': 0.6000000238418579, 'epoch': 2.38}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▉  | 4310/5424 [1:16:31<15:17,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.344, 'grad_norm': 6.037960052490234, 'learning_rate': 1.0192083440496437e-06, 'rewards/margins': 3.5866596698760986, 'rewards/chosen': -4.731215476989746, 'rewards/rejected': -8.31787395477295, 'logps/margins': 17.758634567260742, 'logps/chosen': -8.89856243133545, 'logps/rejected': -26.65719985961914, 'accuracy': 0.800000011920929, 'epoch': 2.38}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████▉  | 4320/5424 [1:16:39<14:57,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.109, 'grad_norm': 3.7051351070404053, 'learning_rate': 1.0017514693257779e-06, 'rewards/margins': 0.8290125131607056, 'rewards/chosen': -4.597362518310547, 'rewards/rejected': -5.426375389099121, 'logps/margins': 6.662114143371582, 'logps/chosen': -10.990863800048828, 'logps/rejected': -17.652978897094727, 'accuracy': 0.5, 'epoch': 2.39}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████▉  | 4330/5424 [1:16:47<15:10,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.3699, 'grad_norm': 4.05370569229126, 'learning_rate': 9.844287304840637e-07, 'rewards/margins': -1.159432053565979, 'rewards/chosen': -6.451152801513672, 'rewards/rejected': -5.291721820831299, 'logps/margins': -3.271101474761963, 'logps/chosen': -18.056442260742188, 'logps/rejected': -14.785344123840332, 'accuracy': 0.699999988079071, 'epoch': 2.39}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 4340/5424 [1:16:55<14:16,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.3758, 'grad_norm': 2.4399759769439697, 'learning_rate': 9.672407086791852e-07, 'rewards/margins': 0.46580713987350464, 'rewards/chosen': -5.1176252365112305, 'rewards/rejected': -5.583432197570801, 'logps/margins': 4.1320271492004395, 'logps/chosen': -11.918039321899414, 'logps/rejected': -16.050067901611328, 'accuracy': 0.5, 'epoch': 2.4}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 4350/5424 [1:17:04<14:35,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.444, 'grad_norm': 3.521275758743286, 'learning_rate': 9.501879805462494e-07, 'rewards/margins': 8.363133430480957, 'rewards/chosen': -5.8553853034973145, 'rewards/rejected': -14.21851634979248, 'logps/margins': 34.82992935180664, 'logps/chosen': -15.068220138549805, 'logps/rejected': -49.89814376831055, 'accuracy': 0.8999999761581421, 'epoch': 2.41}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 4360/5424 [1:17:12<14:30,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.2179, 'grad_norm': 4.146786689758301, 'learning_rate': 9.332711181814408e-07, 'rewards/margins': 1.1116756200790405, 'rewards/chosen': -6.000068664550781, 'rewards/rejected': -7.1117448806762695, 'logps/margins': 4.690340995788574, 'logps/chosen': -15.252037048339844, 'logps/rejected': -19.942378997802734, 'accuracy': 0.5, 'epoch': 2.41}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████  | 4370/5424 [1:17:20<14:09,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.507, 'grad_norm': 4.78935432434082, 'learning_rate': 9.164906891228315e-07, 'rewards/margins': 0.04101228713989258, 'rewards/chosen': -4.694891452789307, 'rewards/rejected': -4.735903739929199, 'logps/margins': 3.468693494796753, 'logps/chosen': -11.236715316772461, 'logps/rejected': -14.705408096313477, 'accuracy': 0.5, 'epoch': 2.42}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████  | 4380/5424 [1:17:28<14:03,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.2378, 'grad_norm': 4.663005352020264, 'learning_rate': 8.998472563313342e-07, 'rewards/margins': 3.0770440101623535, 'rewards/chosen': -5.220193862915039, 'rewards/rejected': -8.297237396240234, 'logps/margins': 16.175861358642578, 'logps/chosen': -12.303511619567871, 'logps/rejected': -28.4793701171875, 'accuracy': 0.800000011920929, 'epoch': 2.42}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████  | 4390/5424 [1:17:36<13:53,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.1013, 'grad_norm': 2.6957104206085205, 'learning_rate': 8.833413781718269e-07, 'rewards/margins': 0.21753016114234924, 'rewards/chosen': -5.611361503601074, 'rewards/rejected': -5.828891277313232, 'logps/margins': 2.122554063796997, 'logps/chosen': -15.275070190429688, 'logps/rejected': -17.397624969482422, 'accuracy': 0.699999988079071, 'epoch': 2.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████  | 4400/5424 [1:17:44<13:40,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.4237, 'grad_norm': 5.903314590454102, 'learning_rate': 8.669736083944074e-07, 'rewards/margins': 2.115478038787842, 'rewards/chosen': -6.354010581970215, 'rewards/rejected': -8.469490051269531, 'logps/margins': 8.063203811645508, 'logps/chosen': -17.956192016601562, 'logps/rejected': -26.019397735595703, 'accuracy': 0.6000000238418579, 'epoch': 2.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████▏ | 4410/5424 [1:17:52<13:05,  1.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.7991, 'grad_norm': 4.047199726104736, 'learning_rate': 8.50744496115829e-07, 'rewards/margins': 2.6324501037597656, 'rewards/chosen': -4.506157398223877, 'rewards/rejected': -7.138607025146484, 'logps/margins': 12.432446479797363, 'logps/chosen': -10.955129623413086, 'logps/rejected': -23.387575149536133, 'accuracy': 0.699999988079071, 'epoch': 2.44}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████▏ | 4420/5424 [1:18:01<13:59,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.1149, 'grad_norm': 4.6932759284973145, 'learning_rate': 8.346545858010641e-07, 'rewards/margins': 0.9908250570297241, 'rewards/chosen': -6.1339521408081055, 'rewards/rejected': -7.124777317047119, 'logps/margins': 1.8321704864501953, 'logps/chosen': -19.026424407958984, 'logps/rejected': -20.858596801757812, 'accuracy': 0.699999988079071, 'epoch': 2.44}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 4430/5424 [1:18:09<13:26,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.508, 'grad_norm': 4.916280269622803, 'learning_rate': 8.187044172450526e-07, 'rewards/margins': 0.38933855295181274, 'rewards/chosen': -5.551215648651123, 'rewards/rejected': -5.940554141998291, 'logps/margins': 2.0253047943115234, 'logps/chosen': -13.823451042175293, 'logps/rejected': -15.848756790161133, 'accuracy': 0.5, 'epoch': 2.45}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 4440/5424 [1:18:17<12:42,  1.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.4333, 'grad_norm': 9.146607398986816, 'learning_rate': 8.028945255545812e-07, 'rewards/margins': 1.879201889038086, 'rewards/chosen': -5.3857855796813965, 'rewards/rejected': -7.264986991882324, 'logps/margins': 9.64775562286377, 'logps/chosen': -12.732441902160645, 'logps/rejected': -22.380197525024414, 'accuracy': 0.6000000238418579, 'epoch': 2.46}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 4450/5424 [1:18:25<12:47,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.196, 'grad_norm': 0.47017714381217957, 'learning_rate': 7.872254411303392e-07, 'rewards/margins': -0.29815658926963806, 'rewards/chosen': -6.603916168212891, 'rewards/rejected': -6.305759906768799, 'logps/margins': 0.4113626480102539, 'logps/chosen': -17.66546058654785, 'logps/rejected': -18.07682228088379, 'accuracy': 0.5, 'epoch': 2.46}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 4460/5424 [1:18:33<12:58,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.1286, 'grad_norm': 9.652735710144043, 'learning_rate': 7.716976896491169e-07, 'rewards/margins': -1.579236626625061, 'rewards/chosen': -6.775060176849365, 'rewards/rejected': -5.195823669433594, 'logps/margins': -5.403950214385986, 'logps/chosen': -18.24454116821289, 'logps/rejected': -12.84058952331543, 'accuracy': 0.20000000298023224, 'epoch': 2.47}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 4470/5424 [1:18:41<12:34,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.4913, 'grad_norm': 3.4507791996002197, 'learning_rate': 7.563117920461754e-07, 'rewards/margins': 0.2661421298980713, 'rewards/chosen': -4.7254204750061035, 'rewards/rejected': -4.991562366485596, 'logps/margins': 1.0133159160614014, 'logps/chosen': -11.677950859069824, 'logps/rejected': -12.691267013549805, 'accuracy': 0.5, 'epoch': 2.47}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 4480/5424 [1:18:49<12:34,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.9157, 'grad_norm': 3.9310545921325684, 'learning_rate': 7.410682644977696e-07, 'rewards/margins': 2.0289814472198486, 'rewards/chosen': -5.560352325439453, 'rewards/rejected': -7.589333534240723, 'logps/margins': 12.386560440063477, 'logps/chosen': -12.425326347351074, 'logps/rejected': -24.8118896484375, 'accuracy': 0.6000000238418579, 'epoch': 2.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 4490/5424 [1:18:57<12:29,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.367, 'grad_norm': 5.274538040161133, 'learning_rate': 7.259676184038289e-07, 'rewards/margins': 1.5691380500793457, 'rewards/chosen': -4.781543731689453, 'rewards/rejected': -6.350680828094482, 'logps/margins': 10.04334545135498, 'logps/chosen': -10.596925735473633, 'logps/rejected': -20.64027214050293, 'accuracy': 0.800000011920929, 'epoch': 2.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 4500/5424 [1:19:05<12:45,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.9918, 'grad_norm': 2.272653102874756, 'learning_rate': 7.110103603707996e-07, 'rewards/margins': 3.3137366771698, 'rewards/chosen': -4.142910957336426, 'rewards/rejected': -7.456646919250488, 'logps/margins': 14.99970817565918, 'logps/chosen': -9.545083999633789, 'logps/rejected': -24.5447940826416, 'accuracy': 0.800000011920929, 'epoch': 2.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 4510/5424 [1:19:13<12:08,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.2904, 'grad_norm': 2.990396499633789, 'learning_rate': 6.961969921946521e-07, 'rewards/margins': 0.05619051307439804, 'rewards/chosen': -6.360517501831055, 'rewards/rejected': -6.416708469390869, 'logps/margins': -0.25928181409835815, 'logps/chosen': -18.79373550415039, 'logps/rejected': -18.534454345703125, 'accuracy': 0.6000000238418579, 'epoch': 2.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 4520/5424 [1:19:22<12:32,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.5478, 'grad_norm': 3.6433658599853516, 'learning_rate': 6.815280108440442e-07, 'rewards/margins': 3.506057024002075, 'rewards/chosen': -5.3065290451049805, 'rewards/rejected': -8.812585830688477, 'logps/margins': 17.771575927734375, 'logps/chosen': -11.14082145690918, 'logps/rejected': -28.912399291992188, 'accuracy': 0.800000011920929, 'epoch': 2.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▎ | 4530/5424 [1:19:30<12:44,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.1385, 'grad_norm': 4.346640586853027, 'learning_rate': 6.670039084436508e-07, 'rewards/margins': 2.3077549934387207, 'rewards/chosen': -5.163516044616699, 'rewards/rejected': -7.4712724685668945, 'logps/margins': 13.36895751953125, 'logps/chosen': -11.27900505065918, 'logps/rejected': -24.64796257019043, 'accuracy': 0.699999988079071, 'epoch': 2.51}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▎ | 4540/5424 [1:19:38<11:58,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.6007, 'grad_norm': 5.540401935577393, 'learning_rate': 6.526251722576516e-07, 'rewards/margins': 1.2939180135726929, 'rewards/chosen': -4.4810566902160645, 'rewards/rejected': -5.774974346160889, 'logps/margins': 5.355494022369385, 'logps/chosen': -10.004329681396484, 'logps/rejected': -15.359823226928711, 'accuracy': 0.699999988079071, 'epoch': 2.51}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 4550/5424 [1:19:46<12:08,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.0937, 'grad_norm': 4.6457839012146, 'learning_rate': 6.383922846733819e-07, 'rewards/margins': 2.1763558387756348, 'rewards/chosen': -5.344653129577637, 'rewards/rejected': -7.5210089683532715, 'logps/margins': 10.211019515991211, 'logps/chosen': -12.23844051361084, 'logps/rejected': -22.449460983276367, 'accuracy': 0.699999988079071, 'epoch': 2.52}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 4560/5424 [1:19:54<11:59,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.3621, 'grad_norm': 5.489541530609131, 'learning_rate': 6.243057231851552e-07, 'rewards/margins': 0.9393669366836548, 'rewards/chosen': -5.990983009338379, 'rewards/rejected': -6.930349826812744, 'logps/margins': 3.032527208328247, 'logps/chosen': -16.101354598999023, 'logps/rejected': -19.133882522583008, 'accuracy': 0.6000000238418579, 'epoch': 2.52}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 4570/5424 [1:20:02<11:07,  1.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.3333, 'grad_norm': 4.186661243438721, 'learning_rate': 6.10365960378238e-07, 'rewards/margins': 1.2776682376861572, 'rewards/chosen': -4.054470062255859, 'rewards/rejected': -5.332138538360596, 'logps/margins': 5.272639274597168, 'logps/chosen': -9.550825119018555, 'logps/rejected': -14.823463439941406, 'accuracy': 0.6000000238418579, 'epoch': 2.53}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 4580/5424 [1:20:10<11:14,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.0569, 'grad_norm': 4.2108378410339355, 'learning_rate': 5.965734639130011e-07, 'rewards/margins': 1.1791824102401733, 'rewards/chosen': -5.71940803527832, 'rewards/rejected': -6.898591041564941, 'logps/margins': 5.237852573394775, 'logps/chosen': -15.392114639282227, 'logps/rejected': -20.629968643188477, 'accuracy': 0.5, 'epoch': 2.53}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▍ | 4590/5424 [1:20:18<10:55,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.1619, 'grad_norm': 2.7355754375457764, 'learning_rate': 5.829286965092213e-07, 'rewards/margins': 0.8169406652450562, 'rewards/chosen': -4.695704460144043, 'rewards/rejected': -5.512645244598389, 'logps/margins': 4.893765449523926, 'logps/chosen': -10.114916801452637, 'logps/rejected': -15.008682250976562, 'accuracy': 0.6000000238418579, 'epoch': 2.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▍ | 4600/5424 [1:20:26<11:05,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.4748, 'grad_norm': 17.856281280517578, 'learning_rate': 5.694321159305694e-07, 'rewards/margins': 0.3575053811073303, 'rewards/chosen': -6.215882778167725, 'rewards/rejected': -6.57338809967041, 'logps/margins': 3.2692153453826904, 'logps/chosen': -15.210453987121582, 'logps/rejected': -18.47966957092285, 'accuracy': 0.4000000059604645, 'epoch': 2.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▍ | 4610/5424 [1:20:34<10:56,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.3913, 'grad_norm': 5.319883823394775, 'learning_rate': 5.560841749692414e-07, 'rewards/margins': 1.574082374572754, 'rewards/chosen': -5.4785566329956055, 'rewards/rejected': -7.052639007568359, 'logps/margins': 8.315760612487793, 'logps/chosen': -13.311185836791992, 'logps/rejected': -21.6269474029541, 'accuracy': 0.6000000238418579, 'epoch': 2.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 4620/5424 [1:20:42<10:35,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 6.6492, 'grad_norm': 3.1436784267425537, 'learning_rate': 5.428853214307772e-07, 'rewards/margins': 2.6433265209198, 'rewards/chosen': -3.5489563941955566, 'rewards/rejected': -6.192282676696777, 'logps/margins': 11.744829177856445, 'logps/chosen': -6.158008575439453, 'logps/rejected': -17.9028377532959, 'accuracy': 0.699999988079071, 'epoch': 2.56}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 4630/5424 [1:20:51<11:44,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.2807, 'grad_norm': 3.2171666622161865, 'learning_rate': 5.298359981190309e-07, 'rewards/margins': 4.132999897003174, 'rewards/chosen': -5.149050712585449, 'rewards/rejected': -9.282050132751465, 'logps/margins': 18.270795822143555, 'logps/chosen': -10.905441284179688, 'logps/rejected': -29.17623519897461, 'accuracy': 0.800000011920929, 'epoch': 2.56}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 4640/5424 [1:20:59<10:42,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.1552, 'grad_norm': 6.401063442230225, 'learning_rate': 5.169366428213191e-07, 'rewards/margins': 0.7311458587646484, 'rewards/chosen': -5.512979984283447, 'rewards/rejected': -6.244126319885254, 'logps/margins': 3.8602452278137207, 'logps/chosen': -13.145855903625488, 'logps/rejected': -17.006101608276367, 'accuracy': 0.5, 'epoch': 2.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 4650/5424 [1:21:07<10:13,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.1078, 'grad_norm': 1.0118845701217651, 'learning_rate': 5.041876882937319e-07, 'rewards/margins': 2.358548879623413, 'rewards/chosen': -5.930700302124023, 'rewards/rejected': -8.289249420166016, 'logps/margins': 11.97659969329834, 'logps/chosen': -15.070098876953125, 'logps/rejected': -27.046701431274414, 'accuracy': 0.699999988079071, 'epoch': 2.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 4660/5424 [1:21:15<10:06,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.6728, 'grad_norm': 3.6015830039978027, 'learning_rate': 4.915895622466166e-07, 'rewards/margins': 4.086765289306641, 'rewards/chosen': -4.732445240020752, 'rewards/rejected': -8.819211959838867, 'logps/margins': 19.310007095336914, 'logps/chosen': -10.360749244689941, 'logps/rejected': -29.67075538635254, 'accuracy': 0.800000011920929, 'epoch': 2.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 4670/5424 [1:21:23<10:01,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.4254, 'grad_norm': 1.3154202699661255, 'learning_rate': 4.791426873302235e-07, 'rewards/margins': -0.1906934231519699, 'rewards/chosen': -6.036168098449707, 'rewards/rejected': -5.845475673675537, 'logps/margins': 6.359812259674072, 'logps/chosen': -13.890057563781738, 'logps/rejected': -20.24987030029297, 'accuracy': 0.6000000238418579, 'epoch': 2.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▋ | 4680/5424 [1:21:31<09:58,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 21.0531, 'grad_norm': 7.449426174163818, 'learning_rate': 4.6684748112053315e-07, 'rewards/margins': 0.8719533681869507, 'rewards/chosen': -6.595134735107422, 'rewards/rejected': -7.467088222503662, 'logps/margins': 3.638312578201294, 'logps/chosen': -18.9746150970459, 'logps/rejected': -22.612926483154297, 'accuracy': 0.699999988079071, 'epoch': 2.59}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▋ | 4690/5424 [1:21:39<09:55,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.1864, 'grad_norm': 1.5567526817321777, 'learning_rate': 4.547043561052433e-07, 'rewards/margins': 3.061371326446533, 'rewards/chosen': -5.502753257751465, 'rewards/rejected': -8.564123153686523, 'logps/margins': 14.345102310180664, 'logps/chosen': -13.47346305847168, 'logps/rejected': -27.818567276000977, 'accuracy': 0.6000000238418579, 'epoch': 2.59}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 4700/5424 [1:21:48<10:25,  1.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.5444, 'grad_norm': 3.6194419860839844, 'learning_rate': 4.4271371966993214e-07, 'rewards/margins': 0.20182828605175018, 'rewards/chosen': -6.1829705238342285, 'rewards/rejected': -6.384798526763916, 'logps/margins': -0.5362547039985657, 'logps/chosen': -17.33389663696289, 'logps/rejected': -16.79764175415039, 'accuracy': 0.699999988079071, 'epoch': 2.6}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 4710/5424 [1:21:56<09:33,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.7447, 'grad_norm': 1.5956979990005493, 'learning_rate': 4.308759740843882e-07, 'rewards/margins': 1.2966951131820679, 'rewards/chosen': -4.891693115234375, 'rewards/rejected': -6.188387870788574, 'logps/margins': 10.910604476928711, 'logps/chosen': -10.57219409942627, 'logps/rejected': -21.482797622680664, 'accuracy': 0.699999988079071, 'epoch': 2.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 4720/5424 [1:22:04<09:36,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.26, 'grad_norm': 7.152553558349609, 'learning_rate': 4.191915164891186e-07, 'rewards/margins': 0.9827299118041992, 'rewards/chosen': -4.401869773864746, 'rewards/rejected': -5.3846001625061035, 'logps/margins': 4.047877311706543, 'logps/chosen': -10.448488235473633, 'logps/rejected': -14.496365547180176, 'accuracy': 0.6000000238418579, 'epoch': 2.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 4730/5424 [1:22:12<09:38,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.8967, 'grad_norm': 3.432770013809204, 'learning_rate': 4.076607388820225e-07, 'rewards/margins': 1.7667487859725952, 'rewards/chosen': -5.176396369934082, 'rewards/rejected': -6.943144798278809, 'logps/margins': 11.606274604797363, 'logps/chosen': -11.715047836303711, 'logps/rejected': -23.32132339477539, 'accuracy': 0.800000011920929, 'epoch': 2.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 4740/5424 [1:22:20<09:09,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.2981, 'grad_norm': 1.135479211807251, 'learning_rate': 3.962840281052427e-07, 'rewards/margins': 1.9212563037872314, 'rewards/chosen': -5.564401149749756, 'rewards/rejected': -7.485658168792725, 'logps/margins': 9.196688652038574, 'logps/chosen': -14.5852689743042, 'logps/rejected': -23.781959533691406, 'accuracy': 0.699999988079071, 'epoch': 2.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 4750/5424 [1:22:29<09:27,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.2322, 'grad_norm': 2.9276788234710693, 'learning_rate': 3.850617658321859e-07, 'rewards/margins': 4.52884578704834, 'rewards/chosen': -5.452654838562012, 'rewards/rejected': -9.981500625610352, 'logps/margins': 23.099031448364258, 'logps/chosen': -11.113743782043457, 'logps/rejected': -34.21277618408203, 'accuracy': 0.8999999761581421, 'epoch': 2.63}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 4760/5424 [1:22:37<09:22,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 22.4574, 'grad_norm': 6.787356853485107, 'learning_rate': 3.7399432855471654e-07, 'rewards/margins': 0.3250180780887604, 'rewards/chosen': -6.349025249481201, 'rewards/rejected': -6.674044132232666, 'logps/margins': -2.0403361320495605, 'logps/chosen': -20.553234100341797, 'logps/rejected': -18.51289939880371, 'accuracy': 0.5, 'epoch': 2.63}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 4770/5424 [1:22:45<09:04,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.7752, 'grad_norm': 3.366245985031128, 'learning_rate': 3.630820875705304e-07, 'rewards/margins': 0.0509578213095665, 'rewards/chosen': -4.975567817687988, 'rewards/rejected': -5.026525497436523, 'logps/margins': 0.935673713684082, 'logps/chosen': -12.587830543518066, 'logps/rejected': -13.523504257202148, 'accuracy': 0.5, 'epoch': 2.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 4780/5424 [1:22:53<08:28,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.5605, 'grad_norm': 1.5076372623443604, 'learning_rate': 3.52325408970694e-07, 'rewards/margins': -0.2285434901714325, 'rewards/chosen': -4.618068218231201, 'rewards/rejected': -4.389524936676025, 'logps/margins': 1.2550415992736816, 'logps/chosen': -11.211782455444336, 'logps/rejected': -12.46682357788086, 'accuracy': 0.5, 'epoch': 2.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 4790/5424 [1:23:01<08:32,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.6998, 'grad_norm': 6.027799606323242, 'learning_rate': 3.417246536273661e-07, 'rewards/margins': 0.5523090958595276, 'rewards/chosen': -6.7404584884643555, 'rewards/rejected': -7.292766571044922, 'logps/margins': 2.6068127155303955, 'logps/chosen': -19.213132858276367, 'logps/rejected': -21.8199462890625, 'accuracy': 0.6000000238418579, 'epoch': 2.65}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 4800/5424 [1:23:09<08:21,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.1032, 'grad_norm': 3.9067440032958984, 'learning_rate': 3.312801771816859e-07, 'rewards/margins': 0.7078164219856262, 'rewards/chosen': -5.751051902770996, 'rewards/rejected': -6.458868503570557, 'logps/margins': 4.656093597412109, 'logps/chosen': -15.368307113647461, 'logps/rejected': -20.02440071105957, 'accuracy': 0.6000000238418579, 'epoch': 2.65}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▊ | 4810/5424 [1:23:17<08:05,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.8108, 'grad_norm': 9.403870582580566, 'learning_rate': 3.209923300318479e-07, 'rewards/margins': -1.0867154598236084, 'rewards/chosen': -5.181302070617676, 'rewards/rejected': -4.094586372375488, 'logps/margins': -6.6998419761657715, 'logps/chosen': -14.984039306640625, 'logps/rejected': -8.284197807312012, 'accuracy': 0.5, 'epoch': 2.66}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 4820/5424 [1:23:25<08:12,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.5029, 'grad_norm': 8.04417896270752, 'learning_rate': 3.1086145732134144e-07, 'rewards/margins': 0.8849164843559265, 'rewards/chosen': -5.105767726898193, 'rewards/rejected': -5.99068546295166, 'logps/margins': 2.9693922996520996, 'logps/chosen': -13.009452819824219, 'logps/rejected': -15.978845596313477, 'accuracy': 0.699999988079071, 'epoch': 2.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 4830/5424 [1:23:34<07:57,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.5994, 'grad_norm': 5.163772106170654, 'learning_rate': 3.0088789892737613e-07, 'rewards/margins': 0.7328530550003052, 'rewards/chosen': -4.680263519287109, 'rewards/rejected': -5.413116931915283, 'logps/margins': 3.8042492866516113, 'logps/chosen': -11.220789909362793, 'logps/rejected': -15.025041580200195, 'accuracy': 0.6000000238418579, 'epoch': 2.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 4840/5424 [1:23:42<08:04,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.7383, 'grad_norm': 1.6962145566940308, 'learning_rate': 2.9107198944947347e-07, 'rewards/margins': 0.7554429769515991, 'rewards/chosen': -5.44982385635376, 'rewards/rejected': -6.205267429351807, 'logps/margins': 5.043122291564941, 'logps/chosen': -12.621927261352539, 'logps/rejected': -17.665050506591797, 'accuracy': 0.4000000059604645, 'epoch': 2.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 4850/5424 [1:23:50<07:39,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.8525, 'grad_norm': 2.635967969894409, 'learning_rate': 2.814140581982483e-07, 'rewards/margins': 1.93890380859375, 'rewards/chosen': -4.990048885345459, 'rewards/rejected': -6.928952217102051, 'logps/margins': 10.631292343139648, 'logps/chosen': -10.149210929870605, 'logps/rejected': -20.78050422668457, 'accuracy': 0.6000000238418579, 'epoch': 2.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|████████▉ | 4860/5424 [1:23:58<07:29,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.7482, 'grad_norm': 2.2106356620788574, 'learning_rate': 2.7191442918435675e-07, 'rewards/margins': 1.0102218389511108, 'rewards/chosen': -6.060329437255859, 'rewards/rejected': -7.07055139541626, 'logps/margins': 3.7750244140625, 'logps/chosen': -16.494897842407227, 'logps/rejected': -20.269920349121094, 'accuracy': 0.6000000238418579, 'epoch': 2.69}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|████████▉ | 4870/5424 [1:24:06<07:27,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.8757, 'grad_norm': 6.469077110290527, 'learning_rate': 2.6257342110762583e-07, 'rewards/margins': 1.8517287969589233, 'rewards/chosen': -4.867060661315918, 'rewards/rejected': -6.718789577484131, 'logps/margins': 13.003725051879883, 'logps/chosen': -9.330537796020508, 'logps/rejected': -22.33426284790039, 'accuracy': 0.699999988079071, 'epoch': 2.69}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|████████▉ | 4880/5424 [1:24:14<07:20,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 20.504, 'grad_norm': 2.5470869541168213, 'learning_rate': 2.533913473463634e-07, 'rewards/margins': -0.39941662549972534, 'rewards/chosen': -6.7066850662231445, 'rewards/rejected': -6.3072686195373535, 'logps/margins': 0.33626461029052734, 'logps/chosen': -18.21232032775879, 'logps/rejected': -18.548585891723633, 'accuracy': 0.4000000059604645, 'epoch': 2.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 4890/5424 [1:24:22<07:15,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.0126, 'grad_norm': 3.5279407501220703, 'learning_rate': 2.443685159468434e-07, 'rewards/margins': 2.424294948577881, 'rewards/chosen': -5.171067237854004, 'rewards/rejected': -7.595362663269043, 'logps/margins': 11.30897045135498, 'logps/chosen': -13.714920043945312, 'logps/rejected': -25.02389144897461, 'accuracy': 0.6000000238418579, 'epoch': 2.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 4900/5424 [1:24:31<07:08,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.1415, 'grad_norm': 8.378293991088867, 'learning_rate': 2.3550522961297284e-07, 'rewards/margins': 0.02535431459546089, 'rewards/chosen': -4.990159034729004, 'rewards/rejected': -5.0155134201049805, 'logps/margins': 0.7280173301696777, 'logps/chosen': -12.623311996459961, 'logps/rejected': -13.35132884979248, 'accuracy': 0.6000000238418579, 'epoch': 2.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 4910/5424 [1:24:39<06:50,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 22.8032, 'grad_norm': 7.923358917236328, 'learning_rate': 2.2680178569613631e-07, 'rewards/margins': -1.1801159381866455, 'rewards/chosen': -6.629746437072754, 'rewards/rejected': -5.449629783630371, 'logps/margins': -7.918736934661865, 'logps/chosen': -20.614795684814453, 'logps/rejected': -12.69605827331543, 'accuracy': 0.10000000149011612, 'epoch': 2.72}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 4920/5424 [1:24:47<06:48,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.0257, 'grad_norm': 2.8151657581329346, 'learning_rate': 2.1825847618521522e-07, 'rewards/margins': 3.5284531116485596, 'rewards/chosen': -6.491568565368652, 'rewards/rejected': -10.020021438598633, 'logps/margins': 18.208389282226562, 'logps/chosen': -15.341156005859375, 'logps/rejected': -33.54954147338867, 'accuracy': 0.800000011920929, 'epoch': 2.72}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 4930/5424 [1:24:55<06:49,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.924, 'grad_norm': 3.6107747554779053, 'learning_rate': 2.0987558769680173e-07, 'rewards/margins': 0.0820738822221756, 'rewards/chosen': -6.208426475524902, 'rewards/rejected': -6.290501117706299, 'logps/margins': 1.454589605331421, 'logps/chosen': -16.352832794189453, 'logps/rejected': -17.80742073059082, 'accuracy': 0.6000000238418579, 'epoch': 2.73}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 4940/5424 [1:25:03<06:32,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.9154, 'grad_norm': 6.673713684082031, 'learning_rate': 2.01653401465573e-07, 'rewards/margins': 1.0540454387664795, 'rewards/chosen': -6.043461799621582, 'rewards/rejected': -7.097507476806641, 'logps/margins': 7.53751277923584, 'logps/chosen': -14.948265075683594, 'logps/rejected': -22.48577880859375, 'accuracy': 0.6000000238418579, 'epoch': 2.73}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████▏| 4950/5424 [1:25:11<06:15,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.8103, 'grad_norm': 6.080226421356201, 'learning_rate': 1.9359219333486335e-07, 'rewards/margins': 0.3009865880012512, 'rewards/chosen': -5.969379425048828, 'rewards/rejected': -6.270366191864014, 'logps/margins': 5.686368942260742, 'logps/chosen': -14.33277702331543, 'logps/rejected': -20.019145965576172, 'accuracy': 0.6000000238418579, 'epoch': 2.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████▏| 4960/5424 [1:25:20<06:28,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.3818, 'grad_norm': 3.041794776916504, 'learning_rate': 1.8569223374740707e-07, 'rewards/margins': 1.2667099237442017, 'rewards/chosen': -5.220586776733398, 'rewards/rejected': -6.487297058105469, 'logps/margins': 6.6572465896606445, 'logps/chosen': -12.911054611206055, 'logps/rejected': -19.568302154541016, 'accuracy': 0.5, 'epoch': 2.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 4970/5424 [1:25:28<06:20,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.0102, 'grad_norm': 3.5418624877929688, 'learning_rate': 1.7795378773626526e-07, 'rewards/margins': -0.036531947553157806, 'rewards/chosen': -5.708921432495117, 'rewards/rejected': -5.672389030456543, 'logps/margins': 0.16426515579223633, 'logps/chosen': -14.818881034851074, 'logps/rejected': -14.983146667480469, 'accuracy': 0.699999988079071, 'epoch': 2.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 4980/5424 [1:25:36<06:02,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.9763, 'grad_norm': 7.324751853942871, 'learning_rate': 1.70377114915935e-07, 'rewards/margins': 1.5360043048858643, 'rewards/chosen': -5.191939353942871, 'rewards/rejected': -6.727943420410156, 'logps/margins': 6.948600769042969, 'logps/chosen': -13.577829360961914, 'logps/rejected': -20.526430130004883, 'accuracy': 0.6000000238418579, 'epoch': 2.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 4990/5424 [1:25:44<06:00,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 21.158, 'grad_norm': 3.692668914794922, 'learning_rate': 1.6296246947363981e-07, 'rewards/margins': 0.04163646697998047, 'rewards/chosen': -6.4495673179626465, 'rewards/rejected': -6.491204261779785, 'logps/margins': -1.040167212486267, 'logps/chosen': -19.01785659790039, 'logps/rejected': -17.97768783569336, 'accuracy': 0.6000000238418579, 'epoch': 2.76}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.2698, 'grad_norm': 1.6519005298614502, 'learning_rate': 1.5571010016080246e-07, 'rewards/margins': 0.570163369178772, 'rewards/chosen': -4.566872596740723, 'rewards/rejected': -5.137035846710205, 'logps/margins': 1.4978857040405273, 'logps/chosen': -11.979998588562012, 'logps/rejected': -13.477884292602539, 'accuracy': 0.699999988079071, 'epoch': 2.77}"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 5000/5424 [1:25:52<05:43,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 5010/5424 [1:26:00<05:32,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.5281, 'grad_norm': 5.540091037750244, 'learning_rate': 1.4862025028469674e-07, 'rewards/margins': 0.5379093289375305, 'rewards/chosen': -5.8183274269104, 'rewards/rejected': -6.356237411499023, 'logps/margins': 3.5874435901641846, 'logps/chosen': -15.660947799682617, 'logps/rejected': -19.248390197753906, 'accuracy': 0.6000000238418579, 'epoch': 2.77}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 5020/5424 [1:26:08<05:25,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.8319, 'grad_norm': 7.322820663452148, 'learning_rate': 1.4169315770028948e-07, 'rewards/margins': 2.7361063957214355, 'rewards/chosen': -5.120993614196777, 'rewards/rejected': -7.8570990562438965, 'logps/margins': 14.80412483215332, 'logps/chosen': -11.989946365356445, 'logps/rejected': -26.7940731048584, 'accuracy': 0.800000011920929, 'epoch': 2.78}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 5030/5424 [1:26:16<05:15,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.2106, 'grad_norm': 7.000107765197754, 'learning_rate': 1.3492905480225692e-07, 'rewards/margins': 1.1201709508895874, 'rewards/chosen': -4.685105323791504, 'rewards/rejected': -5.805276393890381, 'logps/margins': 5.728642463684082, 'logps/chosen': -11.304502487182617, 'logps/rejected': -17.033143997192383, 'accuracy': 0.699999988079071, 'epoch': 2.78}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 5040/5424 [1:26:24<05:08,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 17.2765, 'grad_norm': 2.8003532886505127, 'learning_rate': 1.2832816851719e-07, 'rewards/margins': 1.473198652267456, 'rewards/chosen': -5.720920562744141, 'rewards/rejected': -7.194118499755859, 'logps/margins': 8.248628616333008, 'logps/chosen': -15.5896577835083, 'logps/rejected': -23.838285446166992, 'accuracy': 0.6000000238418579, 'epoch': 2.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 5050/5424 [1:26:33<05:05,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.6201, 'grad_norm': 3.87414288520813, 'learning_rate': 1.2189072029598127e-07, 'rewards/margins': 1.8740589618682861, 'rewards/chosen': -5.788069248199463, 'rewards/rejected': -7.662127494812012, 'logps/margins': 7.71002721786499, 'logps/chosen': -15.659124374389648, 'logps/rejected': -23.369152069091797, 'accuracy': 0.6000000238418579, 'epoch': 2.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 5060/5424 [1:26:41<04:51,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.3442, 'grad_norm': 3.3462257385253906, 'learning_rate': 1.1561692610639441e-07, 'rewards/margins': 3.3223037719726562, 'rewards/chosen': -5.259458541870117, 'rewards/rejected': -8.581762313842773, 'logps/margins': 15.985128402709961, 'logps/chosen': -13.595481872558594, 'logps/rejected': -29.580608367919922, 'accuracy': 0.6000000238418579, 'epoch': 2.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 5070/5424 [1:26:49<05:03,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 13.6781, 'grad_norm': 3.7476229667663574, 'learning_rate': 1.0950699642582108e-07, 'rewards/margins': 3.130683422088623, 'rewards/chosen': -5.477718830108643, 'rewards/rejected': -8.60840129852295, 'logps/margins': 14.425939559936523, 'logps/chosen': -12.868807792663574, 'logps/rejected': -27.29474449157715, 'accuracy': 0.5, 'epoch': 2.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▎| 5080/5424 [1:26:57<04:43,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.1588, 'grad_norm': 9.875548362731934, 'learning_rate': 1.0356113623421648e-07, 'rewards/margins': 1.3741546869277954, 'rewards/chosen': -6.204636096954346, 'rewards/rejected': -7.578790187835693, 'logps/margins': 6.10225772857666, 'logps/chosen': -16.538501739501953, 'logps/rejected': -22.64076042175293, 'accuracy': 0.5, 'epoch': 2.81}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 5090/5424 [1:27:05<04:31,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.6355, 'grad_norm': 4.174144744873047, 'learning_rate': 9.777954500722553e-08, 'rewards/margins': 2.5500781536102295, 'rewards/chosen': -4.697118282318115, 'rewards/rejected': -7.247197151184082, 'logps/margins': 10.866933822631836, 'logps/chosen': -9.102516174316406, 'logps/rejected': -19.969449996948242, 'accuracy': 0.699999988079071, 'epoch': 2.82}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 5100/5424 [1:27:13<04:12,  1.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 6.89, 'grad_norm': 1.2127472162246704, 'learning_rate': 9.216241670948811e-08, 'rewards/margins': 4.048264026641846, 'rewards/chosen': -3.543043851852417, 'rewards/rejected': -7.591307640075684, 'logps/margins': 17.119522094726562, 'logps/chosen': -6.4861297607421875, 'logps/rejected': -23.605653762817383, 'accuracy': 0.800000011920929, 'epoch': 2.82}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 5110/5424 [1:27:21<04:07,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.8985, 'grad_norm': 0.0, 'learning_rate': 8.670993978813436e-08, 'rewards/margins': 2.0724401473999023, 'rewards/chosen': -4.267392158508301, 'rewards/rejected': -6.339831829071045, 'logps/margins': 9.363595962524414, 'logps/chosen': -8.358409881591797, 'logps/rejected': -17.72200584411621, 'accuracy': 0.6000000238418579, 'epoch': 2.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 5120/5424 [1:27:30<04:11,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.2053, 'grad_norm': 2.643629550933838, 'learning_rate': 8.142229716646133e-08, 'rewards/margins': 1.206007957458496, 'rewards/chosen': -7.463980197906494, 'rewards/rejected': -8.669988632202148, 'logps/margins': 8.53565788269043, 'logps/chosen': -18.065887451171875, 'logps/rejected': -26.601543426513672, 'accuracy': 0.6000000238418579, 'epoch': 2.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▍| 5130/5424 [1:27:38<04:08,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.2195, 'grad_norm': 4.258399963378906, 'learning_rate': 7.629966623779461e-08, 'rewards/margins': -0.0434599407017231, 'rewards/chosen': -6.035393714904785, 'rewards/rejected': -5.991934299468994, 'logps/margins': 1.8351243734359741, 'logps/chosen': -14.604032516479492, 'logps/rejected': -16.439159393310547, 'accuracy': 0.4000000059604645, 'epoch': 2.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▍| 5140/5424 [1:27:46<03:57,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.8784, 'grad_norm': 1.5585052967071533, 'learning_rate': 7.134221885954029e-08, 'rewards/margins': 2.7642247676849365, 'rewards/chosen': -4.414055824279785, 'rewards/rejected': -7.178279876708984, 'logps/margins': 11.640719413757324, 'logps/chosen': -9.486757278442383, 'logps/rejected': -21.127477645874023, 'accuracy': 0.800000011920929, 'epoch': 2.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▍| 5150/5424 [1:27:55<03:44,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 21.554, 'grad_norm': 7.348199844360352, 'learning_rate': 6.655012134741678e-08, 'rewards/margins': -0.23658359050750732, 'rewards/chosen': -7.073357582092285, 'rewards/rejected': -6.836773872375488, 'logps/margins': 0.060306645929813385, 'logps/chosen': -19.61427879333496, 'logps/rejected': -19.67458724975586, 'accuracy': 0.5, 'epoch': 2.85}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 5160/5424 [1:28:03<03:33,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.888, 'grad_norm': 3.0382518768310547, 'learning_rate': 6.192353446987597e-08, 'rewards/margins': 2.136159658432007, 'rewards/chosen': -5.120884895324707, 'rewards/rejected': -7.257044792175293, 'logps/margins': 12.662378311157227, 'logps/chosen': -10.303129196166992, 'logps/rejected': -22.96550750732422, 'accuracy': 0.800000011920929, 'epoch': 2.85}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 5170/5424 [1:28:11<03:30,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 18.8873, 'grad_norm': 2.1539053916931152, 'learning_rate': 5.7462613442710315e-08, 'rewards/margins': 0.6682723760604858, 'rewards/chosen': -6.272551536560059, 'rewards/rejected': -6.940824031829834, 'logps/margins': 1.8704427480697632, 'logps/chosen': -17.33821678161621, 'logps/rejected': -19.208660125732422, 'accuracy': 0.800000011920929, 'epoch': 2.86}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 5180/5424 [1:28:20<03:25,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 9.8446, 'grad_norm': 2.269944429397583, 'learning_rate': 5.316750792384417e-08, 'rewards/margins': 1.8649861812591553, 'rewards/chosen': -4.141818523406982, 'rewards/rejected': -6.006804466247559, 'logps/margins': 9.186184883117676, 'logps/chosen': -9.120759963989258, 'logps/rejected': -18.306947708129883, 'accuracy': 0.800000011920929, 'epoch': 2.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 5190/5424 [1:28:28<03:11,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.8757, 'grad_norm': 3.781952381134033, 'learning_rate': 4.9038362008313465e-08, 'rewards/margins': 0.5636928677558899, 'rewards/chosen': -5.5138726234436035, 'rewards/rejected': -6.0775651931762695, 'logps/margins': 6.262924671173096, 'logps/chosen': -13.630348205566406, 'logps/rejected': -19.893274307250977, 'accuracy': 0.6000000238418579, 'epoch': 2.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 5200/5424 [1:28:36<03:00,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.1399, 'grad_norm': 1.639036774635315, 'learning_rate': 4.507531422343281e-08, 'rewards/margins': 2.0073227882385254, 'rewards/chosen': -4.297759532928467, 'rewards/rejected': -6.30508279800415, 'logps/margins': 8.35844612121582, 'logps/chosen': -9.595372200012207, 'logps/rejected': -17.95381736755371, 'accuracy': 0.699999988079071, 'epoch': 2.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 5210/5424 [1:28:44<02:50,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.2205, 'grad_norm': 2.0880823135375977, 'learning_rate': 4.127849752414648e-08, 'rewards/margins': 2.134305953979492, 'rewards/chosen': -5.282494068145752, 'rewards/rejected': -7.416800022125244, 'logps/margins': 8.437520027160645, 'logps/chosen': -13.195833206176758, 'logps/rejected': -21.63335418701172, 'accuracy': 0.699999988079071, 'epoch': 2.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 5220/5424 [1:28:52<02:45,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.5822, 'grad_norm': 4.868936061859131, 'learning_rate': 3.764803928856808e-08, 'rewards/margins': 2.751659870147705, 'rewards/chosen': -5.572932243347168, 'rewards/rejected': -8.324590682983398, 'logps/margins': 10.840719223022461, 'logps/chosen': -14.696060180664062, 'logps/rejected': -25.53677749633789, 'accuracy': 0.6000000238418579, 'epoch': 2.89}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▋| 5230/5424 [1:29:00<02:38,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.9585, 'grad_norm': 3.2846503257751465, 'learning_rate': 3.418406131370844e-08, 'rewards/margins': 1.286213755607605, 'rewards/chosen': -4.6147661209106445, 'rewards/rejected': -5.900980472564697, 'logps/margins': 6.909060478210449, 'logps/chosen': -11.095495223999023, 'logps/rejected': -18.00455665588379, 'accuracy': 0.699999988079071, 'epoch': 2.89}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 5240/5424 [1:29:08<02:26,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.3897, 'grad_norm': 4.021058559417725, 'learning_rate': 3.088667981138826e-08, 'rewards/margins': 1.5795419216156006, 'rewards/chosen': -6.306778907775879, 'rewards/rejected': -7.886321067810059, 'logps/margins': 7.6943159103393555, 'logps/chosen': -17.49086570739746, 'logps/rejected': -25.185182571411133, 'accuracy': 0.5, 'epoch': 2.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 5250/5424 [1:29:16<02:20,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.8518, 'grad_norm': 3.550997495651245, 'learning_rate': 2.7756005404340202e-08, 'rewards/margins': 2.2555761337280273, 'rewards/chosen': -4.196794033050537, 'rewards/rejected': -6.452370643615723, 'logps/margins': 10.317266464233398, 'logps/chosen': -8.340925216674805, 'logps/rejected': -18.658191680908203, 'accuracy': 0.800000011920929, 'epoch': 2.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 5260/5424 [1:29:25<02:14,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.0193, 'grad_norm': 2.483057975769043, 'learning_rate': 2.4792143122495672e-08, 'rewards/margins': 3.4039547443389893, 'rewards/chosen': -4.94607400894165, 'rewards/rejected': -8.350028991699219, 'logps/margins': 15.497518539428711, 'logps/chosen': -11.333860397338867, 'logps/rejected': -26.831375122070312, 'accuracy': 0.699999988079071, 'epoch': 2.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 5270/5424 [1:29:33<02:03,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 12.0806, 'grad_norm': 4.531867980957031, 'learning_rate': 2.1995192399464905e-08, 'rewards/margins': 1.4444881677627563, 'rewards/chosen': -5.024722576141357, 'rewards/rejected': -6.469210624694824, 'logps/margins': 9.241544723510742, 'logps/chosen': -10.90531063079834, 'logps/rejected': -20.146854400634766, 'accuracy': 0.5, 'epoch': 2.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 5280/5424 [1:29:41<02:01,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.7052, 'grad_norm': 11.670575141906738, 'learning_rate': 1.9620723801035966e-08, 'rewards/margins': nan, 'rewards/chosen': nan, 'rewards/rejected': -7.141213893890381, 'logps/margins': nan, 'logps/chosen': nan, 'logps/rejected': -20.148523330688477, 'accuracy': 0.4000000059604645, 'epoch': 2.92}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 5290/5424 [1:29:49<01:47,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.9327, 'grad_norm': 5.3881754875183105, 'learning_rate': 1.7141158921624712e-08, 'rewards/margins': 2.1329519748687744, 'rewards/chosen': -4.21532678604126, 'rewards/rejected': -6.348278045654297, 'logps/margins': 11.130145072937012, 'logps/chosen': -8.117694854736328, 'logps/rejected': -19.24784278869629, 'accuracy': 0.699999988079071, 'epoch': 2.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 5300/5424 [1:29:58<01:43,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.9042, 'grad_norm': 2.514002561569214, 'learning_rate': 1.4828762281297038e-08, 'rewards/margins': 2.24139404296875, 'rewards/chosen': -4.889131546020508, 'rewards/rejected': -7.1305251121521, 'logps/margins': 11.417134284973145, 'logps/chosen': -10.270051956176758, 'logps/rejected': -21.687183380126953, 'accuracy': 0.6000000238418579, 'epoch': 2.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 5310/5424 [1:30:06<01:37,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 6.4468, 'grad_norm': 1.8299660682678223, 'learning_rate': 1.2683611457862677e-08, 'rewards/margins': 3.344355821609497, 'rewards/chosen': -3.6200473308563232, 'rewards/rejected': -6.964402675628662, 'logps/margins': 18.002819061279297, 'logps/chosen': -6.213431358337402, 'logps/rejected': -24.216251373291016, 'accuracy': 0.8999999761581421, 'epoch': 2.94}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 5320/5424 [1:30:14<01:22,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 8.9995, 'grad_norm': 1.6435401439666748, 'learning_rate': 1.0705778418258484e-08, 'rewards/margins': 1.7091853618621826, 'rewards/chosen': -3.971553087234497, 'rewards/rejected': -5.680738925933838, 'logps/margins': 8.517385482788086, 'logps/chosen': -8.520668029785156, 'logps/rejected': -17.03805160522461, 'accuracy': 0.800000011920929, 'epoch': 2.94}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 5330/5424 [1:30:22<01:18,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.9982, 'grad_norm': 3.9624485969543457, 'learning_rate': 8.8953295161337e-09, 'rewards/margins': 0.346245676279068, 'rewards/chosen': -6.113825798034668, 'rewards/rejected': -6.460070610046387, 'logps/margins': 0.7142907381057739, 'logps/chosen': -18.194734573364258, 'logps/rejected': -18.90902328491211, 'accuracy': 0.6000000238418579, 'epoch': 2.95}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 5340/5424 [1:30:31<01:08,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 16.3119, 'grad_norm': 8.038603782653809, 'learning_rate': 7.252325489623402e-09, 'rewards/margins': 2.7793116569519043, 'rewards/chosen': -6.381978988647461, 'rewards/rejected': -9.161291122436523, 'logps/margins': 14.421579360961914, 'logps/chosen': -14.980731010437012, 'logps/rejected': -29.40230941772461, 'accuracy': 0.699999988079071, 'epoch': 2.95}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▊| 5350/5424 [1:30:39<01:01,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.6638, 'grad_norm': 3.4966230392456055, 'learning_rate': 5.776821459311243e-09, 'rewards/margins': 4.744187355041504, 'rewards/chosen': -4.620431900024414, 'rewards/rejected': -9.364618301391602, 'logps/margins': 21.507923126220703, 'logps/chosen': -10.055356979370117, 'logps/rejected': -31.563282012939453, 'accuracy': 0.699999988079071, 'epoch': 2.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▉| 5360/5424 [1:30:48<00:55,  1.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.0002, 'grad_norm': 5.946770668029785, 'learning_rate': 4.468866926382043e-09, 'rewards/margins': 4.001452445983887, 'rewards/chosen': -4.867033004760742, 'rewards/rejected': -8.868485450744629, 'logps/margins': 19.675155639648438, 'logps/chosen': -10.905723571777344, 'logps/rejected': -30.580881118774414, 'accuracy': 1.0, 'epoch': 2.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▉| 5370/5424 [1:30:56<00:43,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 14.6983, 'grad_norm': 2.8876664638519287, 'learning_rate': 3.328505770957002e-09, 'rewards/margins': 3.3279635906219482, 'rewards/chosen': -5.731371879577637, 'rewards/rejected': -9.059335708618164, 'logps/margins': 14.670628547668457, 'logps/chosen': -14.159292221069336, 'logps/rejected': -28.829919815063477, 'accuracy': 0.699999988079071, 'epoch': 2.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▉| 5380/5424 [1:31:04<00:35,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.0433, 'grad_norm': 3.9470293521881104, 'learning_rate': 2.355776250625441e-09, 'rewards/margins': 2.168729066848755, 'rewards/chosen': -5.530701637268066, 'rewards/rejected': -7.6994309425354, 'logps/margins': 8.915159225463867, 'logps/chosen': -13.697854995727539, 'logps/rejected': -22.61301612854004, 'accuracy': 0.6000000238418579, 'epoch': 2.98}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▉| 5390/5424 [1:31:12<00:28,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 10.831, 'grad_norm': 4.543859958648682, 'learning_rate': 1.5507109991602653e-09, 'rewards/margins': 3.086521625518799, 'rewards/chosen': -4.936371803283691, 'rewards/rejected': -8.022892951965332, 'logps/margins': 14.1427001953125, 'logps/chosen': -10.540067672729492, 'logps/rejected': -24.68276596069336, 'accuracy': 0.800000011920929, 'epoch': 2.98}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 5400/5424 [1:31:20<00:19,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 11.825, 'grad_norm': 3.25132155418396, 'learning_rate': 9.133370254221785e-10, 'rewards/margins': 1.673887848854065, 'rewards/chosen': -4.948156833648682, 'rewards/rejected': -6.622045040130615, 'logps/margins': 10.707040786743164, 'logps/chosen': -11.22890853881836, 'logps/rejected': -21.93594741821289, 'accuracy': 0.699999988079071, 'epoch': 2.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 5410/5424 [1:31:29<00:11,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 19.9277, 'grad_norm': 4.354363918304443, 'learning_rate': 4.436757124537394e-10, 'rewards/margins': -0.48746365308761597, 'rewards/chosen': -6.08438777923584, 'rewards/rejected': -5.596923351287842, 'logps/margins': -1.32034170627594, 'logps/chosen': -17.74446678161621, 'logps/rejected': -16.424123764038086, 'accuracy': 0.4000000059604645, 'epoch': 2.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 5420/5424 [1:31:37<00:03,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 15.6695, 'grad_norm': 3.830212354660034, 'learning_rate': 1.417428167632684e-10, 'rewards/margins': 0.9813492894172668, 'rewards/chosen': -5.935286521911621, 'rewards/rejected': -6.9166364669799805, 'logps/margins': 6.194282054901123, 'logps/chosen': -14.491656303405762, 'logps/rejected': -20.68593978881836, 'accuracy': 0.6000000238418579, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5424/5424 [1:31:40<00:00,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating response...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/201 [00:00<?, ?it/s]\u001b[A\n",
      "  1%|          | 2/201 [00:00<01:06,  3.01it/s]\u001b[A\n",
      "  1%|▏         | 3/201 [00:01<01:43,  1.92it/s]\u001b[A\n",
      "  2%|▏         | 4/201 [00:02<01:49,  1.81it/s]\u001b[A\n",
      "  2%|▏         | 5/201 [00:02<01:55,  1.70it/s]\u001b[A\n",
      "  3%|▎         | 6/201 [00:03<01:59,  1.63it/s]\u001b[A\n",
      "  3%|▎         | 7/201 [00:04<02:02,  1.59it/s]\u001b[A\n",
      "  4%|▍         | 8/201 [00:04<02:04,  1.55it/s]\u001b[A\n",
      "  4%|▍         | 9/201 [00:05<02:12,  1.45it/s]\u001b[A\n",
      "  5%|▍         | 10/201 [00:06<02:06,  1.51it/s]\u001b[A\n",
      "  5%|▌         | 11/201 [00:06<02:05,  1.51it/s]\u001b[A\n",
      "  6%|▌         | 12/201 [00:07<02:06,  1.50it/s]\u001b[A\n",
      "  6%|▋         | 13/201 [00:08<02:11,  1.43it/s]\u001b[A\n",
      "  7%|▋         | 14/201 [00:08<02:08,  1.46it/s]\u001b[A\n",
      "  7%|▋         | 15/201 [00:09<02:07,  1.46it/s]\u001b[A\n",
      "  8%|▊         | 16/201 [00:10<02:05,  1.47it/s]\u001b[A\n",
      "  8%|▊         | 17/201 [00:11<02:10,  1.41it/s]\u001b[A\n",
      "  9%|▉         | 18/201 [00:11<02:09,  1.41it/s]\u001b[A\n",
      "  9%|▉         | 19/201 [00:12<02:08,  1.41it/s]\u001b[A\n",
      " 10%|▉         | 20/201 [00:13<02:05,  1.44it/s]\u001b[A\n",
      " 10%|█         | 21/201 [00:13<02:09,  1.39it/s]\u001b[A\n",
      " 11%|█         | 22/201 [00:14<02:08,  1.40it/s]\u001b[A\n",
      " 11%|█▏        | 23/201 [00:15<02:11,  1.35it/s]\u001b[A\n",
      " 12%|█▏        | 24/201 [00:16<02:06,  1.40it/s]\u001b[A\n",
      " 12%|█▏        | 25/201 [00:16<02:04,  1.42it/s]\u001b[A\n",
      " 13%|█▎        | 26/201 [00:17<02:00,  1.45it/s]\u001b[A\n",
      " 13%|█▎        | 27/201 [00:18<01:58,  1.46it/s]\u001b[A\n",
      " 14%|█▍        | 28/201 [00:18<02:03,  1.40it/s]\u001b[A\n",
      " 14%|█▍        | 29/201 [00:19<02:00,  1.42it/s]\u001b[A\n",
      " 15%|█▍        | 30/201 [00:20<01:58,  1.44it/s]\u001b[A\n",
      " 15%|█▌        | 31/201 [00:20<01:55,  1.47it/s]\u001b[A\n",
      " 16%|█▌        | 32/201 [00:21<01:53,  1.49it/s]\u001b[A\n",
      " 16%|█▋        | 33/201 [00:22<01:58,  1.42it/s]\u001b[A\n",
      " 17%|█▋        | 34/201 [00:22<01:56,  1.44it/s]\u001b[A\n",
      " 17%|█▋        | 35/201 [00:23<01:53,  1.47it/s]\u001b[A\n",
      " 18%|█▊        | 36/201 [00:24<01:52,  1.46it/s]\u001b[A\n",
      " 18%|█▊        | 37/201 [00:24<01:51,  1.47it/s]\u001b[A\n",
      " 19%|█▉        | 38/201 [00:25<01:49,  1.49it/s]\u001b[A\n",
      " 19%|█▉        | 39/201 [00:26<01:49,  1.48it/s]\u001b[A\n",
      " 20%|█▉        | 40/201 [00:26<01:48,  1.49it/s]\u001b[A\n",
      " 20%|██        | 41/201 [00:27<01:44,  1.53it/s]\u001b[A\n",
      " 21%|██        | 42/201 [00:28<01:49,  1.45it/s]\u001b[A\n",
      " 21%|██▏       | 43/201 [00:28<01:47,  1.47it/s]\u001b[A\n",
      " 22%|██▏       | 44/201 [00:29<01:46,  1.48it/s]\u001b[A\n",
      " 22%|██▏       | 45/201 [00:30<01:49,  1.42it/s]\u001b[A\n",
      " 23%|██▎       | 46/201 [00:31<01:47,  1.44it/s]\u001b[A\n",
      " 23%|██▎       | 47/201 [00:31<01:45,  1.45it/s]\u001b[A\n",
      " 24%|██▍       | 48/201 [00:32<01:41,  1.51it/s]\u001b[A\n",
      " 24%|██▍       | 49/201 [00:32<01:38,  1.54it/s]\u001b[A\n",
      " 25%|██▍       | 50/201 [00:33<01:38,  1.53it/s]\u001b[A\n",
      " 25%|██▌       | 51/201 [00:34<01:37,  1.53it/s]\u001b[A\n",
      " 26%|██▌       | 52/201 [00:34<01:37,  1.52it/s]\u001b[A\n",
      " 26%|██▋       | 53/201 [00:35<01:37,  1.51it/s]\u001b[A\n",
      " 27%|██▋       | 54/201 [00:36<01:42,  1.44it/s]\u001b[A\n",
      " 27%|██▋       | 55/201 [00:37<01:39,  1.46it/s]\u001b[A\n",
      " 28%|██▊       | 56/201 [00:37<01:38,  1.47it/s]\u001b[A\n",
      " 28%|██▊       | 57/201 [00:38<01:42,  1.41it/s]\u001b[A\n",
      " 29%|██▉       | 58/201 [00:39<01:40,  1.42it/s]\u001b[A\n",
      " 29%|██▉       | 59/201 [00:39<01:37,  1.45it/s]\u001b[A\n",
      " 30%|██▉       | 60/201 [00:40<01:35,  1.47it/s]\u001b[A\n",
      " 30%|███       | 61/201 [00:41<01:34,  1.48it/s]\u001b[A\n",
      " 31%|███       | 62/201 [00:41<01:34,  1.48it/s]\u001b[A\n",
      " 31%|███▏      | 63/201 [00:42<01:33,  1.47it/s]\u001b[A\n",
      " 32%|███▏      | 64/201 [00:43<01:32,  1.48it/s]\u001b[A\n",
      " 32%|███▏      | 65/201 [00:43<01:32,  1.47it/s]\u001b[A\n",
      " 33%|███▎      | 66/201 [00:44<01:35,  1.41it/s]\u001b[A\n",
      " 33%|███▎      | 67/201 [00:45<01:34,  1.42it/s]\u001b[A\n",
      " 34%|███▍      | 68/201 [00:46<01:32,  1.44it/s]\u001b[A\n",
      " 34%|███▍      | 69/201 [00:46<01:28,  1.49it/s]\u001b[A\n",
      " 35%|███▍      | 70/201 [00:47<01:27,  1.50it/s]\u001b[A\n",
      " 35%|███▌      | 71/201 [00:47<01:27,  1.49it/s]\u001b[A\n",
      " 36%|███▌      | 72/201 [00:48<01:26,  1.50it/s]\u001b[A\n",
      " 36%|███▋      | 73/201 [00:49<01:25,  1.50it/s]\u001b[A\n",
      " 37%|███▋      | 74/201 [00:49<01:24,  1.51it/s]\u001b[A\n",
      " 37%|███▋      | 75/201 [00:50<01:28,  1.43it/s]\u001b[A\n",
      " 38%|███▊      | 76/201 [00:51<01:26,  1.44it/s]\u001b[A\n",
      " 38%|███▊      | 77/201 [00:52<01:24,  1.46it/s]\u001b[A\n",
      " 39%|███▉      | 78/201 [00:52<01:27,  1.40it/s]\u001b[A\n",
      " 39%|███▉      | 79/201 [00:53<01:29,  1.36it/s]\u001b[A\n",
      " 40%|███▉      | 80/201 [00:54<01:30,  1.34it/s]\u001b[A\n",
      " 40%|████      | 81/201 [00:55<01:26,  1.39it/s]\u001b[A\n",
      " 41%|████      | 82/201 [00:55<01:28,  1.35it/s]\u001b[A\n",
      " 41%|████▏     | 83/201 [00:56<01:28,  1.33it/s]\u001b[A\n",
      " 42%|████▏     | 84/201 [00:57<01:24,  1.39it/s]\u001b[A\n",
      " 42%|████▏     | 85/201 [00:57<01:21,  1.43it/s]\u001b[A\n",
      " 43%|████▎     | 86/201 [00:58<01:18,  1.46it/s]\u001b[A\n",
      " 43%|████▎     | 87/201 [00:59<01:17,  1.48it/s]\u001b[A\n",
      " 44%|████▍     | 88/201 [00:59<01:15,  1.49it/s]\u001b[A\n",
      " 44%|████▍     | 89/201 [01:00<01:19,  1.41it/s]\u001b[A\n",
      " 45%|████▍     | 90/201 [01:01<01:16,  1.45it/s]\u001b[A\n",
      " 45%|████▌     | 91/201 [01:01<01:13,  1.50it/s]\u001b[A\n",
      " 46%|████▌     | 92/201 [01:02<01:16,  1.43it/s]\u001b[A\n",
      " 46%|████▋     | 93/201 [01:03<01:14,  1.45it/s]\u001b[A\n",
      " 47%|████▋     | 94/201 [01:04<01:13,  1.46it/s]\u001b[A\n",
      " 47%|████▋     | 95/201 [01:04<01:12,  1.47it/s]\u001b[A\n",
      " 48%|████▊     | 96/201 [01:05<01:09,  1.52it/s]\u001b[A\n",
      " 48%|████▊     | 97/201 [01:06<01:08,  1.52it/s]\u001b[A\n",
      " 49%|████▉     | 98/201 [01:06<01:07,  1.52it/s]\u001b[A\n",
      " 49%|████▉     | 99/201 [01:07<01:07,  1.51it/s]\u001b[A\n",
      " 50%|████▉     | 100/201 [01:08<01:06,  1.51it/s]\u001b[A\n",
      " 50%|█████     | 101/201 [01:08<01:06,  1.51it/s]\u001b[A\n",
      " 51%|█████     | 102/201 [01:09<01:05,  1.51it/s]\u001b[A\n",
      " 51%|█████     | 103/201 [01:10<01:05,  1.50it/s]\u001b[A\n",
      " 52%|█████▏    | 104/201 [01:10<01:05,  1.48it/s]\u001b[A\n",
      " 52%|█████▏    | 105/201 [01:11<01:05,  1.47it/s]\u001b[A\n",
      " 53%|█████▎    | 106/201 [01:12<01:03,  1.49it/s]\u001b[A\n",
      " 53%|█████▎    | 107/201 [01:12<01:02,  1.50it/s]\u001b[A\n",
      " 54%|█████▎    | 108/201 [01:13<01:01,  1.51it/s]\u001b[A\n",
      " 54%|█████▍    | 109/201 [01:14<01:00,  1.52it/s]\u001b[A\n",
      " 55%|█████▍    | 110/201 [01:14<00:58,  1.56it/s]\u001b[A\n",
      " 55%|█████▌    | 111/201 [01:15<00:58,  1.53it/s]\u001b[A\n",
      " 56%|█████▌    | 112/201 [01:15<00:58,  1.51it/s]\u001b[A\n",
      " 56%|█████▌    | 113/201 [01:16<00:57,  1.52it/s]\u001b[A\n",
      " 57%|█████▋    | 114/201 [01:17<00:57,  1.52it/s]\u001b[A\n",
      " 57%|█████▋    | 115/201 [01:17<00:57,  1.49it/s]\u001b[A\n",
      " 58%|█████▊    | 116/201 [01:18<00:56,  1.51it/s]\u001b[A\n",
      " 58%|█████▊    | 117/201 [01:19<00:55,  1.51it/s]\u001b[A\n",
      " 59%|█████▊    | 118/201 [01:19<00:54,  1.52it/s]\u001b[A\n",
      " 59%|█████▉    | 119/201 [01:20<00:53,  1.52it/s]\u001b[A\n",
      " 60%|█████▉    | 120/201 [01:21<00:53,  1.52it/s]\u001b[A\n",
      " 60%|██████    | 121/201 [01:21<00:52,  1.52it/s]\u001b[A\n",
      " 61%|██████    | 122/201 [01:22<00:52,  1.50it/s]\u001b[A\n",
      " 61%|██████    | 123/201 [01:23<00:54,  1.44it/s]\u001b[A\n",
      " 62%|██████▏   | 124/201 [01:24<00:52,  1.46it/s]\u001b[A\n",
      " 62%|██████▏   | 125/201 [01:24<00:50,  1.51it/s]\u001b[A\n",
      " 63%|██████▎   | 126/201 [01:25<00:48,  1.55it/s]\u001b[A\n",
      " 63%|██████▎   | 127/201 [01:25<00:48,  1.54it/s]\u001b[A\n",
      " 64%|██████▎   | 128/201 [01:26<00:47,  1.54it/s]\u001b[A\n",
      " 64%|██████▍   | 129/201 [01:27<00:46,  1.54it/s]\u001b[A\n",
      " 65%|██████▍   | 130/201 [01:27<00:46,  1.53it/s]\u001b[A\n",
      " 65%|██████▌   | 131/201 [01:28<00:45,  1.53it/s]\u001b[A\n",
      " 66%|██████▌   | 132/201 [01:29<00:49,  1.39it/s]\u001b[A\n",
      " 66%|██████▌   | 133/201 [01:30<00:48,  1.40it/s]\u001b[A\n",
      " 67%|██████▋   | 134/201 [01:30<00:46,  1.44it/s]\u001b[A\n",
      " 67%|██████▋   | 135/201 [01:31<00:45,  1.46it/s]\u001b[A\n",
      " 68%|██████▊   | 136/201 [01:32<00:44,  1.48it/s]\u001b[A\n",
      " 68%|██████▊   | 137/201 [01:32<00:43,  1.48it/s]\u001b[A\n",
      " 69%|██████▊   | 138/201 [01:33<00:42,  1.48it/s]\u001b[A\n",
      " 69%|██████▉   | 139/201 [01:34<00:41,  1.50it/s]\u001b[A\n",
      " 70%|██████▉   | 140/201 [01:34<00:40,  1.50it/s]\u001b[A\n",
      " 70%|███████   | 141/201 [01:35<00:40,  1.50it/s]\u001b[A\n",
      " 71%|███████   | 142/201 [01:36<00:41,  1.43it/s]\u001b[A\n",
      " 71%|███████   | 143/201 [01:36<00:40,  1.44it/s]\u001b[A\n",
      " 72%|███████▏  | 144/201 [01:37<00:39,  1.45it/s]\u001b[A\n",
      " 72%|███████▏  | 145/201 [01:38<00:37,  1.48it/s]\u001b[A\n",
      " 73%|███████▎  | 146/201 [01:38<00:38,  1.42it/s]\u001b[A\n",
      " 73%|███████▎  | 147/201 [01:39<00:39,  1.36it/s]\u001b[A\n",
      " 74%|███████▎  | 148/201 [01:40<00:37,  1.41it/s]\u001b[A\n",
      " 74%|███████▍  | 149/201 [01:41<00:36,  1.43it/s]\u001b[A\n",
      " 75%|███████▍  | 150/201 [01:41<00:35,  1.45it/s]\u001b[A\n",
      " 75%|███████▌  | 151/201 [01:42<00:36,  1.39it/s]\u001b[A\n",
      " 76%|███████▌  | 152/201 [01:43<00:34,  1.41it/s]\u001b[A\n",
      " 76%|███████▌  | 153/201 [01:43<00:33,  1.43it/s]\u001b[A\n",
      " 77%|███████▋  | 154/201 [01:44<00:32,  1.46it/s]\u001b[A\n",
      " 77%|███████▋  | 155/201 [01:45<00:31,  1.47it/s]\u001b[A\n",
      " 78%|███████▊  | 156/201 [01:45<00:30,  1.49it/s]\u001b[A\n",
      " 78%|███████▊  | 157/201 [01:46<00:29,  1.47it/s]\u001b[A\n",
      " 79%|███████▊  | 158/201 [01:47<00:28,  1.48it/s]\u001b[A\n",
      " 79%|███████▉  | 159/201 [01:47<00:28,  1.50it/s]\u001b[A\n",
      " 80%|███████▉  | 160/201 [01:48<00:27,  1.51it/s]\u001b[A\n",
      " 80%|████████  | 161/201 [01:49<00:26,  1.52it/s]\u001b[A\n",
      " 81%|████████  | 162/201 [01:49<00:25,  1.52it/s]\u001b[A\n",
      " 81%|████████  | 163/201 [01:50<00:25,  1.51it/s]\u001b[A\n",
      " 82%|████████▏ | 164/201 [01:51<00:24,  1.51it/s]\u001b[A\n",
      " 82%|████████▏ | 165/201 [01:51<00:24,  1.48it/s]\u001b[A\n",
      " 83%|████████▎ | 166/201 [01:52<00:23,  1.48it/s]\u001b[A\n",
      " 83%|████████▎ | 167/201 [01:53<00:22,  1.48it/s]\u001b[A\n",
      " 84%|████████▎ | 168/201 [01:54<00:23,  1.40it/s]\u001b[A\n",
      " 84%|████████▍ | 169/201 [01:54<00:22,  1.42it/s]\u001b[A\n",
      " 85%|████████▍ | 170/201 [01:55<00:21,  1.44it/s]\u001b[A\n",
      " 85%|████████▌ | 171/201 [01:56<00:20,  1.46it/s]\u001b[A\n",
      " 86%|████████▌ | 172/201 [01:56<00:20,  1.40it/s]\u001b[A\n",
      " 86%|████████▌ | 173/201 [01:57<00:19,  1.43it/s]\u001b[A\n",
      " 87%|████████▋ | 174/201 [01:58<00:18,  1.45it/s]\u001b[A\n",
      " 87%|████████▋ | 175/201 [01:58<00:17,  1.46it/s]\u001b[A\n",
      " 88%|████████▊ | 176/201 [01:59<00:16,  1.48it/s]\u001b[A\n",
      " 88%|████████▊ | 177/201 [02:00<00:16,  1.48it/s]\u001b[A\n",
      " 89%|████████▊ | 178/201 [02:00<00:15,  1.48it/s]\u001b[A\n",
      " 89%|████████▉ | 179/201 [02:01<00:14,  1.50it/s]\u001b[A\n",
      " 90%|████████▉ | 180/201 [02:02<00:13,  1.51it/s]\u001b[A\n",
      " 90%|█████████ | 181/201 [02:02<00:13,  1.51it/s]\u001b[A\n",
      " 91%|█████████ | 182/201 [02:03<00:12,  1.51it/s]\u001b[A\n",
      " 91%|█████████ | 183/201 [02:04<00:11,  1.52it/s]\u001b[A\n",
      " 92%|█████████▏| 184/201 [02:04<00:11,  1.51it/s]\u001b[A\n",
      " 92%|█████████▏| 185/201 [02:05<00:10,  1.51it/s]\u001b[A\n",
      " 93%|█████████▎| 186/201 [02:06<00:09,  1.50it/s]\u001b[A\n",
      " 93%|█████████▎| 187/201 [02:06<00:09,  1.50it/s]\u001b[A\n",
      " 94%|█████████▎| 188/201 [02:07<00:08,  1.49it/s]\u001b[A\n",
      " 94%|█████████▍| 189/201 [02:08<00:08,  1.49it/s]\u001b[A\n",
      " 95%|█████████▍| 190/201 [02:08<00:07,  1.49it/s]\u001b[A\n",
      " 95%|█████████▌| 191/201 [02:09<00:06,  1.50it/s]\u001b[A\n",
      " 96%|█████████▌| 192/201 [02:10<00:05,  1.51it/s]\u001b[A\n",
      " 96%|█████████▌| 193/201 [02:10<00:05,  1.50it/s]\u001b[A\n",
      " 97%|█████████▋| 194/201 [02:11<00:04,  1.54it/s]\u001b[A\n",
      " 97%|█████████▋| 195/201 [02:12<00:03,  1.54it/s]\u001b[A\n",
      " 98%|█████████▊| 196/201 [02:12<00:03,  1.52it/s]\u001b[A\n",
      " 98%|█████████▊| 197/201 [02:13<00:02,  1.51it/s]\u001b[A\n",
      " 99%|█████████▊| 198/201 [02:14<00:01,  1.52it/s]\u001b[A\n",
      " 99%|█████████▉| 199/201 [02:14<00:01,  1.52it/s]\u001b[A\n",
      "100%|█████████▉| 200/201 [02:15<00:00,  1.44it/s]\u001b[A\n",
      "                                                     \n",
      "100%|██████████| 5424/5424 [1:34:22<00:00,  1.20it/s]\n",
      "100%|██████████| 201/201 [02:16<00:00,  1.47it/s]\u001b[A\n",
      "                                                 \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 14.567987442016602, 'eval_accuracy': 0.5970149253731343, 'eval_runtime': 161.6752, 'eval_samples_per_second': 1.243, 'eval_steps_per_second': 1.243, 'eval_rewards/margins': 1.2954055070877075, 'eval_rewards/chosen': -5.490493297576904, 'eval_rewards/rejected': -6.785898208618164, 'eval_logps/margins': 7.8321757316589355, 'eval_logps/chosen': -13.313322067260742, 'eval_logps/rejected': -21.145496368408203, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5424/5424 [1:34:32<00:00,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_runtime': 5672.4826, 'train_samples_per_second': 0.956, 'train_steps_per_second': 0.956, 'train_loss': 14.85964111029926, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Language Modeling: Run MODPO on helpful preferences, with the safe reward as margin, to train language models that are both helpful and safe\n",
    "# r = (w)r_better + (1-w)r_safer\n",
    "%run scripts/modpo/beavertails/mocpo.py \\\n",
    "    --sft_model_name \"haoranxu/X-ALMA-13B-Pretrain\" \\\n",
    "    --margin_reward_model_name_1 \"../dist/CPO-kiwi/checkpoint-2009\" \\\n",
    "    --margin_reward_model_name_2 \"../dist/CPO-xcomet/checkpoint-2009\" \\\n",
    "    --prompt_template \"Translate this from Russian to English:\\nRussian: {raw_prompt} \\nEnglish:\" \\\n",
    "    --dataset_name \"marulyanova/PKU-SafeRLHF-10K-Modified-fluency\" \\\n",
    "    --sanity_check False \\\n",
    "    --w1 0.33 \\\n",
    "    --w2 0.33 \\\n",
    "    --max_length 512 \\\n",
    "    --peft_config.r 64 \\\n",
    "    --peft_config.target_modules q_proj k_proj v_proj o_proj \\\n",
    "    --peft_config.lora_alpha 1 \\\n",
    "    --peft_config.lora_dropout 0\n",
    "    # --training_args.output_dir \"./output/PKU-Alignment/PKU-SafeRLHF-10K/modpo/lm/($w)better+(1-$w)safer\" \\\n",
    "    # --training_args.run_name PKU-Alignment/PKU-SafeRLHF-10K/modpo/lm/($w)better+(1-$w)safer \\\n",
    "    # --training_args.per_device_train_batch_size 1 \\\n",
    "    # --training_args.per_device_eval_batch_size 1 \\\n",
    "    # --training_args.gradient_accumulation_steps 1 \\\n",
    "    # --training_args.learning_rate 5e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "001e9e1d-fb0e-4b6c-9d65-385c3c7ae09b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-17T19:20:02.609829Z",
     "iopub.status.busy": "2024-10-17T19:20:02.609494Z",
     "iopub.status.idle": "2024-10-17T19:20:02.656525Z",
     "shell.execute_reply": "2024-10-17T19:20:02.655743Z",
     "shell.execute_reply.started": "2024-10-17T19:20:02.609804Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e38cf83-e154-4ddb-b94e-dedafdf6b48e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DataSphere Kernel",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}