Experiment_1 (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7049be18-2025-4ed0-b06f-b0269fbdbb93",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from transformers import BertTokenizer\n",
    "import pickle\n",
    "\n",
    "from textattack.models.wrappers import ModelWrapper\n",
    "from textattack.datasets import HuggingFaceDataset\n",
    "from textattack.attack_recipes import PWWSRen2019\n",
    "from textattack import Attacker, AttackArgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9033dc5d-ad95-400b-95af-79a713169aac",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.nn import CrossEntropyLoss\n",
    "import textattack\n",
    "from model_wrapper import ModelWrapper\n",
    "\n",
    "from models import model_1 as mm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d1a81f4e-44c5-4d2f-a422-c6f8f4e2b81c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42ac3741-10da-4013-8925-1f0d8ce411a7",
   "metadata": {
    "tags": []
   },
   "source": [
    "### PyTorchModelWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3916c7f2-9acc-4cf6-9447-1cb42d824813",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class PyTorchModelWrapper(ModelWrapper):\n",
    "    \"\"\"Loads a PyTorch model (`nn.Module`) and tokenizer.\n",
    "\n",
    "    Args:\n",
    "        model (torch.nn.Module): PyTorch model\n",
    "        tokenizer: tokenizer whose output can be packed as a tensor and passed to the model.\n",
    "            No type requirement, but most have `tokenizer` method that accepts list of strings.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, model, tokenizer):\n",
    "        if not isinstance(model, torch.nn.Module):\n",
    "            raise TypeError(\n",
    "                f\"PyTorch model must be torch.nn.Module, got type {type(model)}\"\n",
    "            )\n",
    "\n",
    "        self.model = model\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "    def to(self, device):\n",
    "        self.model.to(device)\n",
    "\n",
    "    def __call__(self, text_input_list, batch_size=32):\n",
    "        model_device = next(self.model.parameters()).device\n",
    "        ############################################################ Change 1\n",
    "        ids = self.tokenizer(text_input_list, return_tensors=\"pt\", padding=True)\n",
    "        ids = ids.to(model_device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            outputs = textattack.shared.utils.batch_model_predict(\n",
    "                self.model, ids, batch_size=batch_size\n",
    "            )\n",
    "\n",
    "        return outputs\n",
    "\n",
    "    def get_grad(self, text_input, loss_fn=CrossEntropyLoss()):\n",
    "        \"\"\"Get gradient of loss with respect to input tokens.\n",
    "\n",
    "        Args:\n",
    "            text_input (str): input string\n",
    "            loss_fn (torch.nn.Module): loss function. Default is `torch.nn.CrossEntropyLoss`\n",
    "        Returns:\n",
    "            Dict of ids, tokens, and gradient as numpy array.\n",
    "        \"\"\"\n",
    "\n",
    "        if not hasattr(self.model, \"get_input_embeddings\"):\n",
    "            raise AttributeError(\n",
    "                f\"{type(self.model)} must have method `get_input_embeddings` that returns `torch.nn.Embedding` object that represents input embedding layer\"\n",
    "            )\n",
    "        if not isinstance(loss_fn, torch.nn.Module):\n",
    "            raise ValueError(\"Loss function must be of type `torch.nn.Module`.\")\n",
    "\n",
    "        self.model.train()\n",
    "\n",
    "        embedding_layer = self.model.get_input_embeddings()\n",
    "        original_state = embedding_layer.weight.requires_grad\n",
    "        embedding_layer.weight.requires_grad = True\n",
    "\n",
    "        emb_grads = []\n",
    "\n",
    "        def grad_hook(module, grad_in, grad_out):\n",
    "            emb_grads.append(grad_out[0])\n",
    "\n",
    "        emb_hook = embedding_layer.register_backward_hook(grad_hook)\n",
    "\n",
    "        self.model.zero_grad()\n",
    "        model_device = next(self.model.parameters()).device\n",
    "        \n",
    "        ######################################################### Change 2\n",
    "        ids = self.tokenizer(text_input_list, return_tensors=\"pt\", padding=True)\n",
    "        ids = ids.to(model_device)\n",
    "\n",
    "        predictions = self.model(ids)\n",
    "\n",
    "        output = predictions.argmax(dim=1)\n",
    "        loss = loss_fn(predictions, output)\n",
    "        loss.backward()\n",
    "\n",
    "        # grad w.r.t to word embeddings\n",
    "\n",
    "        # Fix for Issue #601\n",
    "\n",
    "        # Check if gradient has shape [max_sequence,1,_] ( when model input in transpose of input sequence)\n",
    "\n",
    "        if emb_grads[0].shape[1] == 1:\n",
    "            grad = torch.transpose(emb_grads[0], 0, 1)[0].cpu().numpy()\n",
    "        else:\n",
    "            # gradient has shape [1,max_sequence,_]\n",
    "            grad = emb_grads[0][0].cpu().numpy()\n",
    "\n",
    "        embedding_layer.weight.requires_grad = original_state\n",
    "        emb_hook.remove()\n",
    "        self.model.eval()\n",
    "\n",
    "        output = {\"ids\": ids[0].tolist(), \"gradient\": grad}\n",
    "\n",
    "        return output\n",
    "\n",
    "    def _tokenize(self, inputs):\n",
    "        \"\"\"Helper method that for `tokenize`\n",
    "        Args:\n",
    "            inputs (list[str]): list of input strings\n",
    "        Returns:\n",
    "            tokens (list[list[str]]): List of list of tokens as strings\n",
    "        \"\"\"\n",
    "        return [self.tokenizer.convert_ids_to_tokens(self.tokenizer(x)) for x in inputs]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0580a9c-44b6-4bc9-9cd1-5bcdb0c19f15",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Self Tx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a031a345-3045-460a-b645-722db0226563",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "d_model = 768\n",
    "max_token = 512\n",
    "nhead = 12\n",
    "nlayer = 6\n",
    "device = \"cuda\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d85c977d-8aba-460e-afb0-474b636ee2e1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(\"bert_embedding.pkl\", \"rb\") as file:\n",
    "    embd = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f2e89fd2-a358-4b23-b041-32c94e2ca1b5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = mm.ad_transformer(d_model,nhead, nlayer, max_token, embd, device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "89254143-a950-4897-aba7-4d3cdca9e44f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b504c8db-b6e6-4f29-a188-2f2cc26e63fc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load(\"weight_trained_1.pth\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8ce8152-23ec-49bd-a94b-a40a13654076",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Spam Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bbbadc2d-fed9-4268-b05c-d511694bff8b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#f = pd.read_csv(\"dataset/spam_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ec987be9-14c2-434c-b029-bc29f04eb71d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df = df.drop(\"len\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "09fc61b8-bb0e-4f75-b0da-debb5aa1bc2d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "### Removing Sequence more than 500 tokens\n",
    "#tok_threshold = []\n",
    "#for i in range(len(df) - 1, -1, -1):\n",
    "#    tok_len = len(tokenizer(df.iloc[i,1], return_tensors=\"pt\", padding=True)[\"input_ids\"][0])\n",
    "#    if tok_len > 500:\n",
    "#        tok_threshold.append(i)\n",
    "#        \n",
    "#df.drop(859, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6a7e333a-a3cf-4095-bdfe-2b18ad513b50",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df = df[df[\"text_type\"] == \"spam\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "047cca20-5893-4e61-9293-aca91a1577e6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df_test.to_csv(\"attack_spam_only.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3aa6e39e-a4ae-4645-a807-d123ef095cdb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(\"dataset/attack_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ac673a5a-d863-4828-9d6b-c894d30b31be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_test = df_test.replace(\"spam\",0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5946d094-a653-4cdb-bfd8-274627f15105",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Testing Acurracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "cca8c6be-5ed1-4d6f-a837-8ead76728b98",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## Actual Value (SPAM, HAM)  ## Y - Predicted Value (SPAM, HAM)\n",
    "#conf_matrix = np.zeros((2,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "63c28c47-cda0-491c-b7d0-35d5116fe270",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfor i in range(len(df_test)):\\n    input_list = df_test.iloc[i,1]\\n    tok_input = tokenizer(input_list, return_tensors=\"pt\", padding=True).to(device)\\n    out = model(tok_input)\\n    \\n    index_1 = torch.argmax(out[0]).tolist()\\n    \\n    if df_test.iloc[i,0] == 0:\\n        index_2 = 0\\n    else:\\n        index_2 = 1\\n        \\n    conf_matrix[index_1][index_2] += 1\\n'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "for i in range(len(df_test)):\n",
    "    input_list = df_test.iloc[i,1]\n",
    "    tok_input = tokenizer(input_list, return_tensors=\"pt\", padding=True).to(device)\n",
    "    out = model(tok_input)\n",
    "    \n",
    "    index_1 = torch.argmax(out[0]).tolist()\n",
    "    \n",
    "    if df_test.iloc[i,0] == 0:\n",
    "        index_2 = 0\n",
    "    else:\n",
    "        index_2 = 1\n",
    "        \n",
    "    conf_matrix[index_1][index_2] += 1\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "afaccf29-a4ba-4cd2-b595-2c6643b4ee03",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#conf_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a61c33c6-8611-41cb-88e6-e07b450f23b8",
   "metadata": {},
   "source": [
    "### Attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7e2b0de6-f07d-4d1f-a52f-c93d988500e0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_test = df_test.rename(columns={'text_type': 'label'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a7a14008-a471-4843-9de0-2bc4b10a7ed2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_wrapper = PyTorchModelWrapper(model, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "85db722d-9611-4aaa-98d0-450653bbd8df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from textattack import Attack\n",
    "from textattack.search_methods import GreedySearch\n",
    "from textattack.constraints.pre_transformation import (\n",
    "    RepeatModification,\n",
    "    StopwordModification,\n",
    ")\n",
    "from textattack.goal_functions import UntargetedClassification, TargetedClassification\n",
    "from textattack.transformations import WordSwapEmbedding\n",
    "from textattack.constraints.pre_transformation import RepeatModification\n",
    "from textattack.constraints.pre_transformation import StopwordModification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c447e937-f2ef-438b-a96b-b402d1dbd13d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n"
     ]
    }
   ],
   "source": [
    "goal_function = TargetedClassification(model_wrapper, target_class = 1)\n",
    "\n",
    "transformation = WordSwapEmbedding()\n",
    "\n",
    "constraints = [RepeatModification(), StopwordModification()]\n",
    "\n",
    "search_method = GreedySearch()\n",
    "\n",
    "attack = Attack(goal_function, constraints, transformation, search_method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "367153a4-ef8c-4f98-81c2-67e37da48e0b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "custom_ds = []\n",
    "for i in range(len(df_test)):\n",
    "    custom_ds.append((df_test.iloc[i,1], int(df_test.iloc[i,0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "415e4dd0-77e1-4795-9d70-4f75309f3f77",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset = textattack.datasets.Dataset(custom_ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad4631a6-626f-4899-ac4a-1a84713bbfd5",
   "metadata": {},
   "source": [
    "### Getting Low Rank of only Attention Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d170a09d-d159-4c0a-a113-5756cb2a0f5f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def svd_sa(model, rank):\n",
    "    all_model_weight = model.state_dict()\n",
    "    \n",
    "    for layer_num in range(nlayer):\n",
    "        layer_name = 'transformer_encoder.layers.' + str(layer_num) + '.self_attn.in_proj_weight'\n",
    "\n",
    "        in_proj_weight = all_model_weight[layer_name]\n",
    "        q, k, v = torch.split(in_proj_weight, [d_model, d_model, d_model])\n",
    "\n",
    "        U, s, V = np.linalg.svd(q.to(\"cpu\").numpy())\n",
    "        low_rank_mat = np.dot(U[:, :rank], np.dot(np.diag(s[:rank]), V[:rank, :]))\n",
    "        q = low_rank_mat\n",
    "\n",
    "        U, s, V = np.linalg.svd(k.to(\"cpu\").numpy())\n",
    "        low_rank_mat = np.dot(U[:, :rank], np.dot(np.diag(s[:rank]), V[:rank, :]))\n",
    "        k = low_rank_mat\n",
    "\n",
    "        U, s, V = np.linalg.svd(v.to(\"cpu\").numpy())\n",
    "        low_rank_mat = np.dot(U[:, :rank], np.dot(np.diag(s[:rank]), V[:rank, :]))\n",
    "        v = low_rank_mat\n",
    "\n",
    "        all_model_weight[layer_name] = torch.concat((torch.tensor(q),torch.tensor(k), torch.tensor(v))).to(\"cuda\")\n",
    "        \n",
    "    return all_model_weight"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0486247-b0b1-417e-93d1-9b18d1a092d3",
   "metadata": {},
   "source": [
    "### Experimenting on different rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b03e61e4-cdab-4535-a035-0d0a9b153326",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "transformation = WordSwapEmbedding()\n",
    "constraints = [RepeatModification(), StopwordModification()]\n",
    "search_method = GreedySearch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08f829d2-61e3-4ce9-bc58-26891d3b5ff5",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_1.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                                                                    | 0/30 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|████▏                                                                                                                       | 1/30 [00:15<07:37, 15.77s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 1 / 0 / 1:   3%|██▎                                                                   | 1/30 [00:15<07:37, 15.79s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 1 / 0 / 1:   7%|████▋                                                                 | 2/30 [00:37<08:46, 18.81s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 2 / 0 / 2:   7%|████▋                                                                 | 2/30 [00:37<08:46, 18.82s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 2 / 0 / 2:  10%|███████                                                               | 3/30 [01:06<09:54, 22.03s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 3 / 0 / 3:  10%|███████                                                               | 3/30 [01:06<09:55, 22.04s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 3 / 0 / 3:  13%|█████████▎                                                            | 4/30 [01:38<10:41, 24.68s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 4 / 0 / 4:  13%|█████████▎                                                            | 4/30 [01:38<10:41, 24.68s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 4 / 0 / 4:  17%|███████████▋                                                          | 5/30 [02:03<10:17, 24.68s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 5 / 0 / 5:  17%|███████████▋                                                          | 5/30 [02:03<10:17, 24.69s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627041310.ta.chkpt\" at 2024-04-20 15:30:41 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 5 / 0 / 5:  20%|██████████████                                                        | 6/30 [02:31<10:07, 25.33s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 6 / 0 / 6:  20%|██████████████                                                        | 6/30 [02:32<10:08, 25.33s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 6 / 0 / 6:  23%|████████████████▎                                                     | 7/30 [03:03<10:04, 26.27s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 7 / 0 / 7:  23%|████████████████▎                                                     | 7/30 [03:03<10:04, 26.27s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 7 / 0 / 7:  27%|██████████████████▋                                                   | 8/30 [03:04<08:27, 23.08s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 8 / 0 / 8:  27%|██████████████████▋                                                   | 8/30 [03:04<08:27, 23.08s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 8 / 0 / 8:  30%|█████████████████████                                                 | 9/30 [03:15<07:36, 21.73s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 9 / 0 / 9:  30%|█████████████████████                                                 | 9/30 [03:15<07:36, 21.74s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 9 / 0 / 9:  33%|███████████████████████                                              | 10/30 [03:35<07:10, 21.51s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 10 / 0 / 10:  33%|██████████████████████▎                                            | 10/30 [03:35<07:10, 21.51s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627132965.ta.chkpt\" at 2024-04-20 15:32:12 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 10 / 0 / 10:  37%|████████████████████████▌                                          | 11/30 [04:06<07:05, 22.41s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 11 / 0 / 11:  37%|████████████████████████▌                                          | 11/30 [04:06<07:05, 22.41s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 11 / 0 / 11:  40%|██████████████████████████▊                                        | 12/30 [04:15<06:23, 21.30s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 12 / 0 / 12:  40%|██████████████████████████▊                                        | 12/30 [04:15<06:23, 21.30s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 12 / 0 / 12:  43%|█████████████████████████████                                      | 13/30 [04:24<05:45, 20.33s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 13 / 0 / 13:  43%|█████████████████████████████                                      | 13/30 [04:24<05:45, 20.33s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 13 / 0 / 13:  47%|███████████████████████████████▎                                   | 14/30 [04:33<05:12, 19.54s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 14 / 0 / 14:  47%|███████████████████████████████▎                                   | 14/30 [04:33<05:12, 19.54s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 14 / 0 / 14:  50%|█████████████████████████████████▌                                 | 15/30 [04:42<04:42, 18.81s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 15 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [04:42<04:42, 18.81s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627200079.ta.chkpt\" at 2024-04-20 15:33:20 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 15 / 0 / 15:  53%|███████████████████████████████████▋                               | 16/30 [04:55<04:18, 18.48s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 16 / 0 / 16:  53%|███████████████████████████████████▋                               | 16/30 [04:55<04:18, 18.49s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 16 / 0 / 16:  57%|█████████████████████████████████████▉                             | 17/30 [05:23<04:07, 19.05s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 17 / 0 / 17:  57%|█████████████████████████████████████▉                             | 17/30 [05:23<04:07, 19.05s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 17 / 0 / 17:  60%|████████████████████████████████████████▏                          | 18/30 [05:46<03:50, 19.24s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 18 / 0 / 18:  60%|████████████████████████████████████████▏                          | 18/30 [05:46<03:50, 19.24s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 18 / 0 / 18:  63%|██████████████████████████████████████████▍                        | 19/30 [06:15<03:37, 19.74s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 19 / 0 / 19:  63%|██████████████████████████████████████████▍                        | 19/30 [06:15<03:37, 19.74s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 19 / 0 / 19:  67%|████████████████████████████████████████████▋                      | 20/30 [06:31<03:15, 19.58s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 20 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [06:31<03:15, 19.58s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627309518.ta.chkpt\" at 2024-04-20 15:35:09 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 20 / 0 / 20:  70%|██████████████████████████████████████████████▉                    | 21/30 [06:53<02:57, 19.68s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 21 / 0 / 21:  70%|██████████████████████████████████████████████▉                    | 21/30 [06:53<02:57, 19.68s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 21 / 0 / 21:  73%|█████████████████████████████████████████████████▏                 | 22/30 [07:01<02:33, 19.18s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 22 / 0 / 22:  73%|█████████████████████████████████████████████████▏                 | 22/30 [07:02<02:33, 19.18s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 26 / 0 / 26:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [08:35<00:57, 19.08s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 27 / 0 / 27:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [08:35<00:57, 19.08s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 27 / 0 / 27:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [08:45<00:37, 18.79s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 28 / 0 / 28:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [08:46<00:37, 18.79s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 28 / 0 / 28:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [09:04<00:18, 18.78s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 29 / 0 / 29:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [09:04<00:18, 18.78s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 29 / 0 / 29: 100%|███████████████████████████████████████████████████████████████████| 30/30 [09:13<00:00, 18.45s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 30 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [09:13<00:00, 18.45s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627471304.ta.chkpt\" at 2024-04-20 15:37:51 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 30 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [09:13<00:00, 18.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 0      |\n",
      "| Number of failed attacks:     | 30     |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 100.0% |\n",
      "| Attack success rate:          | 0.0%   |\n",
      "| Average perturbed word %:     | nan%   |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 1150.0 |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "/opt/conda/lib/python3.10/site-packages/textattack/metrics/attack_metrics/words_perturbed.py:83: RuntimeWarning: Mean of empty slice.\n",
      "  average_perc_words_perturbed = self.perturbed_word_percentages.mean()\n",
      "/opt/conda/lib/python3.10/site-packages/numpy/core/_methods.py:129: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_6.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                                                                    | 0/30 [00:00<?, ?it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 1 / 1:   3%|██▎                                                                   | 1/30 [00:00<00:01, 20.69it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 2 / 2:   7%|████▋                                                                 | 2/30 [00:00<00:01, 19.14it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 2 / 2:  10%|███████                                                               | 3/30 [00:03<00:35,  1.31s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 2 / 3:  10%|███████                                                               | 3/30 [00:03<00:35,  1.33s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 3 / 4:  13%|█████████▎                                                            | 4/30 [00:04<00:26,  1.01s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 3 / 5:  17%|███████████▋                                                          | 5/30 [00:07<00:38,  1.55s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627486797.ta.chkpt\" at 2024-04-20 15:38:06 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 3 / 5:  20%|██████████████                                                        | 6/30 [00:11<00:44,  1.87s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 3 / 0 / 3 / 6:  20%|██████████████                                                        | 6/30 [00:11<00:44,  1.87s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 3 / 7:  23%|████████████████▎                                                     | 7/30 [00:14<00:48,  2.13s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 1 / 3 / 8:  27%|██████████████████▋                                                   | 8/30 [00:15<00:43,  1.96s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 1 / 3 / 8:  30%|█████████████████████                                                 | 9/30 [00:17<00:41,  1.98s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 3 / 9:  30%|█████████████████████                                                 | 9/30 [00:17<00:41,  1.98s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 1 / 3 / 10:  33%|██████████████████████▋                                             | 10/30 [00:20<00:41,  2.07s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627499773.ta.chkpt\" at 2024-04-20 15:38:19 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 3 / 11:  37%|████████████████████████▉                                           | 11/30 [00:24<00:42,  2.23s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 3 / 11:  37%|████████████████████████▉                                           | 11/30 [00:31<00:54,  2.86s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 3 / 11:  40%|███████████████████████████▏                                        | 12/30 [00:32<00:49,  2.73s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 1 / 3 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:32<00:49,  2.73s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 1 / 3 / 12:  43%|█████████████████████████████▍                                      | 13/30 [00:41<00:53,  3.17s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 2 / 3 / 13:  43%|█████████████████████████████▍                                      | 13/30 [00:41<00:53,  3.17s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 2 / 3 / 13:  47%|███████████████████████████████▋                                    | 14/30 [00:43<00:49,  3.08s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 3 / 14:  47%|███████████████████████████████▋                                    | 14/30 [00:43<00:49,  3.08s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 3 / 14:  50%|██████████████████████████████████                                  | 15/30 [00:51<00:51,  3.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 3 / 3 / 15:  50%|██████████████████████████████████                                  | 15/30 [00:51<00:51,  3.44s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627530577.ta.chkpt\" at 2024-04-20 15:38:50 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 3 / 3 / 15:  53%|████████████████████████████████████▎                               | 16/30 [01:02<00:54,  3.88s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 3 / 16:  53%|███████████████████████████████████▋                               | 16/30 [01:02<00:54,  3.89s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 3 / 16:  57%|█████████████████████████████████████▉                             | 17/30 [01:06<00:50,  3.89s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 3 / 3 / 17:  57%|█████████████████████████████████████▉                             | 17/30 [01:06<00:50,  3.89s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 3 / 4 / 18:  60%|████████████████████████████████████████▏                          | 18/30 [01:06<00:44,  3.67s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 3 / 4 / 18:  63%|██████████████████████████████████████████▍                        | 19/30 [01:12<00:42,  3.84s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 4 / 19:  63%|██████████████████████████████████████████▍                        | 19/30 [01:12<00:42,  3.84s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 3 / 4 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:20<00:40,  4.01s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627559146.ta.chkpt\" at 2024-04-20 15:39:19 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 3 / 4 / 20:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:25<00:36,  4.09s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 3 / 4 / 21:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:25<00:36,  4.09s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 3 / 4 / 21:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:34<00:34,  4.30s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 4 / 4 / 22:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:34<00:34,  4.31s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 4 / 4 / 22:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:37<00:29,  4.24s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 4 / 4 / 23:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:37<00:29,  4.24s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 4 / 4 / 23:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:46<00:26,  4.46s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 4 / 24:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:47<00:26,  4.46s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 4 / 24:  83%|███████████████████████████████████████████████████████▊           | 25/30 [01:55<00:23,  4.64s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 4 / 4 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [01:55<00:23,  4.64s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627594970.ta.chkpt\" at 2024-04-20 15:39:54 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 4 / 4 / 25:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:11<00:20,  5.08s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 5 / 4 / 26:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:12<00:20,  5.08s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 5 / 4 / 26:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:18<00:15,  5.11s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 5 / 4 / 27:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:18<00:15,  5.12s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 5 / 4 / 27:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:23<00:10,  5.14s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 5 / 4 / 28:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:23<00:10,  5.14s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 5 / 5 / 29:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:23<00:04,  4.96s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 5 / 5 / 29: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:25<00:00,  4.86s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 5 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:25<00:00,  4.86s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627624975.ta.chkpt\" at 2024-04-20 15:40:24 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 5 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:25<00:00,  4.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 20     |\n",
      "| Number of failed attacks:     | 5      |\n",
      "| Number of skipped attacks:    | 5      |\n",
      "| Original accuracy:            | 83.33% |\n",
      "| Accuracy under attack:        | 16.67% |\n",
      "| Attack success rate:          | 80.0%  |\n",
      "| Average perturbed word %:     | 9.25%  |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 361.6  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_11.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                                                                    | 0/30 [00:00<?, ?it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 1 / 1:   3%|██▎                                                                   | 1/30 [00:00<00:01, 21.96it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 2 / 2:   7%|████▋                                                                 | 2/30 [00:00<00:01, 16.50it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 2 / 2:  10%|███████                                                               | 3/30 [00:03<00:33,  1.26s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 2 / 3:  10%|███████                                                               | 3/30 [00:03<00:34,  1.27s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 3 / 4:  13%|█████████▎                                                            | 4/30 [00:03<00:24,  1.04it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 3 / 5:  17%|███████████▋                                                          | 5/30 [00:07<00:35,  1.44s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627640510.ta.chkpt\" at 2024-04-20 15:40:40 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 3 / 5:  20%|██████████████                                                        | 6/30 [00:10<00:42,  1.78s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 3 / 0 / 3 / 6:  20%|██████████████                                                        | 6/30 [00:10<00:42,  1.79s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 3 / 7:  23%|████████████████▎                                                     | 7/30 [00:14<00:47,  2.06s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 1 / 3 / 8:  27%|██████████████████▋                                                   | 8/30 [00:15<00:41,  1.90s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 1 / 3 / 8:  30%|█████████████████████                                                 | 9/30 [00:19<00:44,  2.13s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 3 / 9:  30%|█████████████████████                                                 | 9/30 [00:19<00:44,  2.13s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 1 / 3 / 10:  33%|██████████████████████▋                                             | 10/30 [00:22<00:44,  2.21s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627655401.ta.chkpt\" at 2024-04-20 15:40:55 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 3 / 11:  37%|████████████████████████▉                                           | 11/30 [00:25<00:44,  2.35s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 3 / 11:  40%|███████████████████████████▏                                        | 12/30 [00:35<00:52,  2.94s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 2 / 3 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:35<00:52,  2.94s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 2 / 3 / 12:  43%|█████████████████████████████▍                                      | 13/30 [00:43<00:57,  3.38s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 3 / 3 / 13:  43%|█████████████████████████████▍                                      | 13/30 [00:43<00:57,  3.38s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 3 / 3 / 13:  47%|███████████████████████████████▋                                    | 14/30 [00:45<00:52,  3.28s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 3 / 3 / 14:  47%|███████████████████████████████▋                                    | 14/30 [00:46<00:52,  3.29s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 3 / 3 / 14:  50%|██████████████████████████████████                                  | 15/30 [00:54<00:54,  3.65s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 4 / 3 / 15:  50%|██████████████████████████████████                                  | 15/30 [00:54<00:54,  3.65s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627688057.ta.chkpt\" at 2024-04-20 15:41:28 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 4 / 3 / 15:  53%|████████████████████████████████████▎                               | 16/30 [01:07<00:59,  4.25s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 5 / 3 / 16:  53%|████████████████████████████████████▎                               | 16/30 [01:07<00:59,  4.25s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 5 / 3 / 16:  57%|██████████████████████████████████████▌                             | 17/30 [01:11<00:54,  4.21s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 5 / 3 / 17:  57%|██████████████████████████████████████▌                             | 17/30 [01:11<00:54,  4.21s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 5 / 4 / 18:  60%|████████████████████████████████████████▊                           | 18/30 [01:11<00:47,  3.98s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 5 / 4 / 18:  63%|███████████████████████████████████████████                         | 19/30 [01:18<00:45,  4.13s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 5 / 4 / 19:  63%|██████████████████████████████████████████▍                        | 19/30 [01:18<00:45,  4.13s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 5 / 4 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:25<00:42,  4.27s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627718792.ta.chkpt\" at 2024-04-20 15:41:58 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 5 / 4 / 20:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:28<00:37,  4.22s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 5 / 4 / 21:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:28<00:38,  4.23s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 5 / 4 / 21:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:37<00:35,  4.44s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 6 / 4 / 22:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:37<00:35,  4.44s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 6 / 4 / 22:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:40<00:30,  4.36s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 6 / 4 / 23:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:40<00:30,  4.36s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 6 / 4 / 23:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:49<00:27,  4.56s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 6 / 4 / 24:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:49<00:27,  4.56s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 6 / 4 / 24:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:00<00:24,  4.83s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 6 / 4 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:00<00:24,  4.83s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627754069.ta.chkpt\" at 2024-04-20 15:42:34 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 6 / 4 / 25:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:12<00:20,  5.11s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 6 / 4 / 26:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:12<00:20,  5.11s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 6 / 4 / 26:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:18<00:15,  5.15s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 6 / 4 / 27:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:18<00:15,  5.15s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 6 / 4 / 27:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:27<00:10,  5.28s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 6 / 4 / 28:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:27<00:10,  5.28s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 6 / 4 / 28:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:30<00:05,  5.21s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 6 / 4 / 29:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:31<00:05,  5.21s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 6 / 4 / 29: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:33<00:00,  5.10s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 6 / 4 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:33<00:00,  5.10s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627786420.ta.chkpt\" at 2024-04-20 15:43:06 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 6 / 4 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:33<00:00,  5.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 20     |\n",
      "| Number of failed attacks:     | 6      |\n",
      "| Number of skipped attacks:    | 4      |\n",
      "| Original accuracy:            | 86.67% |\n",
      "| Accuracy under attack:        | 20.0%  |\n",
      "| Attack success rate:          | 76.92% |\n",
      "| Average perturbed word %:     | 8.81%  |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 363.31 |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_16.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                                                                    | 0/30 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|████▏                                                                                                                       | 1/30 [00:02<01:19,  2.74s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 0 / 1:   3%|██▎                                                                   | 1/30 [00:02<01:20,  2.77s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 1 / 2:   7%|████▋                                                                 | 2/30 [00:02<00:39,  1.40s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 1 / 2:  10%|███████                                                               | 3/30 [00:06<00:57,  2.13s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 1 / 3:  10%|███████                                                               | 3/30 [00:06<00:57,  2.14s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 2 / 4:  13%|█████████▎                                                            | 4/30 [00:06<00:41,  1.61s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 2 / 4:  17%|███████████▋                                                          | 5/30 [00:09<00:48,  1.95s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 3 / 0 / 2 / 5:  17%|███████████▋                                                          | 5/30 [00:09<00:48,  1.96s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627803545.ta.chkpt\" at 2024-04-20 15:43:23 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 2 / 6:  20%|██████████████                                                        | 6/30 [00:13<00:53,  2.21s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 2 / 6:  23%|████████████████▎                                                     | 7/30 [00:17<00:56,  2.44s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 2 / 7:  23%|████████████████▎                                                     | 7/30 [00:17<00:56,  2.44s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 2 / 8:  27%|██████████████████▋                                                   | 8/30 [00:17<00:49,  2.24s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 2 / 8:  30%|█████████████████████                                                 | 9/30 [00:20<00:46,  2.23s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 1 / 2 / 9:  30%|█████████████████████                                                 | 9/30 [00:20<00:46,  2.23s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 2 / 10:  33%|██████████████████████▋                                             | 10/30 [00:22<00:45,  2.29s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627816697.ta.chkpt\" at 2024-04-20 15:43:36 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 2 / 10:  37%|████████████████████████▉                                           | 11/30 [00:26<00:46,  2.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 1 / 2 / 11:  37%|████████████████████████▉                                           | 11/30 [00:26<00:46,  2.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 2 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:34<00:51,  2.85s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 2 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:36<00:55,  3.07s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 2 / 12:  43%|█████████████████████████████▍                                      | 13/30 [00:42<00:56,  3.30s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 2 / 13:  43%|█████████████████████████████▍                                      | 13/30 [00:42<00:56,  3.30s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 2 / 13:  47%|███████████████████████████████▋                                    | 14/30 [00:44<00:51,  3.21s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 2 / 2 / 14:  47%|███████████████████████████████▎                                   | 14/30 [00:44<00:51,  3.21s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 2 / 2 / 14:  50%|█████████████████████████████████▌                                 | 15/30 [00:53<00:53,  3.60s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 2 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [00:54<00:54,  3.60s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627847766.ta.chkpt\" at 2024-04-20 15:44:07 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 2 / 15:  53%|███████████████████████████████████▋                               | 16/30 [01:06<00:58,  4.15s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 3 / 2 / 16:  53%|███████████████████████████████████▋                               | 16/30 [01:06<00:58,  4.15s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 3 / 2 / 16:  57%|█████████████████████████████████████▉                             | 17/30 [01:09<00:53,  4.11s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 2 / 17:  57%|█████████████████████████████████████▉                             | 17/30 [01:09<00:53,  4.11s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 3 / 18:  60%|████████████████████████████████████████▏                          | 18/30 [01:09<00:46,  3.89s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 3 / 18:  63%|██████████████████████████████████████████▍                        | 19/30 [01:16<00:44,  4.04s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 3 / 3 / 19:  63%|██████████████████████████████████████████▍                        | 19/30 [01:16<00:44,  4.04s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 3 / 3 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:23<00:41,  4.20s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627877658.ta.chkpt\" at 2024-04-20 15:44:37 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 3 / 3 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:26<00:43,  4.34s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 3 / 3 / 20:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:29<00:38,  4.27s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 3 / 3 / 21:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:29<00:38,  4.27s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 3 / 3 / 21:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:38<00:35,  4.47s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 4 / 3 / 22:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:38<00:35,  4.48s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 4 / 3 / 22:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:43<00:31,  4.51s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 3 / 23:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:43<00:31,  4.51s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 3 / 23:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:51<00:27,  4.65s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 4 / 3 / 24:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:51<00:27,  4.65s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 4 / 3 / 24:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:04<00:24,  5.00s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 4 / 3 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:05<00:25,  5.00s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627918773.ta.chkpt\" at 2024-04-20 15:45:18 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 4 / 3 / 25:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:15<00:20,  5.22s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 4 / 3 / 26:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:15<00:20,  5.22s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 4 / 3 / 26:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:21<00:15,  5.26s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 4 / 3 / 27:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:21<00:15,  5.26s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 4 / 3 / 27:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:32<00:10,  5.45s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 4 / 3 / 28:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:32<00:10,  5.45s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 4 / 3 / 28:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:35<00:05,  5.36s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 4 / 3 / 29:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:35<00:05,  5.36s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 4 / 3 / 29: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:37<00:00,  5.24s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 23 / 4 / 3 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:37<00:00,  5.25s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627951108.ta.chkpt\" at 2024-04-20 15:45:51 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 23 / 4 / 3 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:37<00:00,  5.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 23     |\n",
      "| Number of failed attacks:     | 4      |\n",
      "| Number of skipped attacks:    | 3      |\n",
      "| Original accuracy:            | 90.0%  |\n",
      "| Accuracy under attack:        | 13.33% |\n",
      "| Attack success rate:          | 85.19% |\n",
      "| Average perturbed word %:     | 10.87% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 361.0  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_21.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                                                                    | 0/30 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|████▏                                                                                                                       | 1/30 [00:02<01:19,  2.75s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 0 / 1:   3%|██▎                                                                   | 1/30 [00:02<01:20,  2.77s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 1 / 2:   7%|████▋                                                                 | 2/30 [00:02<00:39,  1.40s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 1 / 2:  10%|███████                                                               | 3/30 [00:06<01:01,  2.27s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 1 / 3:  10%|███████                                                               | 3/30 [00:06<01:01,  2.28s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 2 / 4:  13%|█████████▎                                                            | 4/30 [00:06<00:44,  1.72s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 2 / 4:  17%|███████████▋                                                          | 5/30 [00:10<00:51,  2.06s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 3 / 0 / 2 / 5:  17%|███████████▋                                                          | 5/30 [00:10<00:51,  2.06s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627968388.ta.chkpt\" at 2024-04-20 15:46:08 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 2 / 6:  20%|██████████████                                                        | 6/30 [00:13<00:55,  2.29s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 2 / 6:  23%|████████████████▎                                                     | 7/30 [00:17<00:57,  2.49s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 2 / 7:  23%|████████████████▎                                                     | 7/30 [00:17<00:57,  2.49s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 2 / 8:  27%|██████████████████▋                                                   | 8/30 [00:18<00:50,  2.27s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 2 / 8:  30%|█████████████████████                                                 | 9/30 [00:20<00:47,  2.26s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 1 / 2 / 9:  30%|█████████████████████                                                 | 9/30 [00:20<00:47,  2.26s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 2 / 10:  33%|██████████████████████▋                                             | 10/30 [00:23<00:46,  2.33s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713627981377.ta.chkpt\" at 2024-04-20 15:46:21 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 2 / 10:  37%|████████████████████████▉                                           | 11/30 [00:27<00:46,  2.46s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 1 / 2 / 11:  37%|████████████████████████▉                                           | 11/30 [00:27<00:46,  2.46s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 2 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:34<00:51,  2.87s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 2 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:42<01:03,  3.54s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 2 / 12:  43%|█████████████████████████████▍                                      | 13/30 [00:43<00:56,  3.31s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 2 / 13:  43%|█████████████████████████████▍                                      | 13/30 [00:43<00:56,  3.31s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 2 / 13:  47%|███████████████████████████████▋                                    | 14/30 [00:45<00:51,  3.22s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 2 / 2 / 14:  47%|███████████████████████████████▎                                   | 14/30 [00:45<00:51,  3.22s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 2 / 2 / 14:  50%|█████████████████████████████████▌                                 | 15/30 [00:53<00:53,  3.58s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 2 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [00:53<00:53,  3.58s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628011823.ta.chkpt\" at 2024-04-20 15:46:51 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 2 / 15:  53%|███████████████████████████████████▋                               | 16/30 [01:07<00:59,  4.23s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 4 / 2 / 16:  53%|███████████████████████████████████▋                               | 16/30 [01:07<00:59,  4.23s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 4 / 2 / 16:  57%|█████████████████████████████████████▉                             | 17/30 [01:11<00:54,  4.20s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 2 / 17:  57%|█████████████████████████████████████▉                             | 17/30 [01:11<00:54,  4.20s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 3 / 18:  60%|████████████████████████████████████████▏                          | 18/30 [01:11<00:47,  3.97s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 3 / 18:  63%|██████████████████████████████████████████▍                        | 19/30 [01:18<00:45,  4.12s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 4 / 3 / 19:  63%|██████████████████████████████████████████▍                        | 19/30 [01:18<00:45,  4.13s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 4 / 3 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:25<00:42,  4.27s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628043550.ta.chkpt\" at 2024-04-20 15:47:23 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 4 / 3 / 20:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:31<00:39,  4.38s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 4 / 3 / 21:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:31<00:39,  4.38s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 4 / 3 / 21:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:40<00:36,  4.58s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 5 / 3 / 22:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:40<00:36,  4.58s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 5 / 3 / 22:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:45<00:32,  4.60s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 3 / 23:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:45<00:32,  4.60s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 3 / 23:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:55<00:28,  4.80s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 5 / 3 / 24:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:55<00:28,  4.80s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 5 / 3 / 24:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:03<00:24,  4.96s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 5 / 3 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:04<00:24,  4.96s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628082136.ta.chkpt\" at 2024-04-20 15:48:02 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 5 / 4 / 28:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:38<00:05,  5.45s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 4 / 29:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:38<00:05,  5.45s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 4 / 29: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:41<00:00,  5.39s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 5 / 4 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:41<00:00,  5.40s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628462568.ta.chkpt\" at 2024-04-20 15:54:22 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 5 / 4 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:41<00:00,  5.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 21     |\n",
      "| Number of failed attacks:     | 5      |\n",
      "| Number of skipped attacks:    | 4      |\n",
      "| Original accuracy:            | 86.67% |\n",
      "| Accuracy under attack:        | 16.67% |\n",
      "| Attack success rate:          | 80.77% |\n",
      "| Average perturbed word %:     | 10.73% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 380.04 |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_36.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                                                                    | 0/30 [00:00<?, ?it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 1 / 1:   3%|██▎                                                                   | 1/30 [00:00<00:01, 17.37it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 2 / 2:   7%|████▋                                                                 | 2/30 [00:00<00:01, 18.08it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 2 / 2:  10%|███████                                                               | 3/30 [00:03<00:34,  1.28s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 2 / 3:  10%|███████                                                               | 3/30 [00:03<00:34,  1.29s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 3 / 4:  13%|█████████▎                                                            | 4/30 [00:03<00:25,  1.03it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 3 / 5:  17%|███████████▋                                                          | 5/30 [00:07<00:36,  1.46s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628477129.ta.chkpt\" at 2024-04-20 15:54:37 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 3 / 5:  20%|██████████████                                                        | 6/30 [00:10<00:43,  1.80s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 3 / 0 / 3 / 6:  20%|██████████████                                                        | 6/30 [00:10<00:43,  1.80s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 3 / 7:  23%|████████████████▎                                                     | 7/30 [00:14<00:47,  2.08s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 1 / 3 / 8:  27%|██████████████████▋                                                   | 8/30 [00:15<00:42,  1.92s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 1 / 3 / 8:  30%|█████████████████████                                                 | 9/30 [00:21<00:49,  2.34s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 3 / 9:  30%|█████████████████████                                                 | 9/30 [00:21<00:49,  2.34s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 3 / 9:  33%|███████████████████████                                              | 10/30 [00:23<00:47,  2.40s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 1 / 3 / 10:  33%|██████████████████████▋                                             | 10/30 [00:24<00:48,  2.40s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628493819.ta.chkpt\" at 2024-04-20 15:54:53 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 1 / 3 / 10:  37%|████████████████████████▉                                           | 11/30 [00:27<00:48,  2.53s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 3 / 11:  37%|████████████████████████▉                                           | 11/30 [00:27<00:48,  2.53s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 3 / 11:  40%|███████████████████████████▏                                        | 12/30 [00:31<00:47,  2.62s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 1 / 3 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:31<00:47,  2.63s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 1 / 3 / 12:  43%|█████████████████████████████▍                                      | 13/30 [00:40<00:52,  3.11s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 2 / 3 / 13:  43%|█████████████████████████████▍                                      | 13/30 [00:40<00:52,  3.11s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 2 / 3 / 13:  47%|███████████████████████████████▋                                    | 14/30 [00:43<00:50,  3.14s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 3 / 14:  47%|███████████████████████████████▋                                    | 14/30 [00:44<00:50,  3.14s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 3 / 14:  50%|██████████████████████████████████                                  | 15/30 [00:52<00:52,  3.52s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 3 / 3 / 15:  50%|██████████████████████████████████                                  | 15/30 [00:52<00:52,  3.52s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628522633.ta.chkpt\" at 2024-04-20 15:55:22 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 3 / 3 / 15:  53%|████████████████████████████████████▎                               | 16/30 [01:05<00:57,  4.12s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 3 / 16:  53%|███████████████████████████████████▋                               | 16/30 [01:05<00:57,  4.12s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 3 / 16:  57%|█████████████████████████████████████▉                             | 17/30 [01:09<00:53,  4.09s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 3 / 3 / 17:  57%|█████████████████████████████████████▉                             | 17/30 [01:09<00:53,  4.09s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 3 / 3 / 17:  60%|████████████████████████████████████████▏                          | 18/30 [01:12<00:48,  4.04s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 3 / 18:  60%|████████████████████████████████████████▏                          | 18/30 [01:12<00:48,  4.04s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 3 / 18:  63%|██████████████████████████████████████████▍                        | 19/30 [01:19<00:46,  4.19s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 3 / 3 / 19:  63%|██████████████████████████████████████████▍                        | 19/30 [01:19<00:46,  4.20s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 3 / 3 / 19:  67%|████████████████████████████████████████████▋                      | 20/30 [01:24<00:42,  4.23s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 3 / 3 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:24<00:42,  4.23s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628554434.ta.chkpt\" at 2024-04-20 15:55:54 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 3 / 3 / 20:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:30<00:38,  4.31s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 3 / 3 / 21:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:30<00:38,  4.31s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 3 / 3 / 21:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:39<00:36,  4.54s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 4 / 3 / 22:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:39<00:36,  4.54s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 4 / 3 / 22:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:45<00:32,  4.57s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 3 / 23:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:45<00:32,  4.57s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 3 / 23:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:52<00:28,  4.69s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 4 / 3 / 24:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:52<00:28,  4.70s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 4 / 3 / 24:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:01<00:24,  4.86s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 4 / 3 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:01<00:24,  4.87s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628591436.ta.chkpt\" at 2024-04-20 15:56:31 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 4 / 3 / 25:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:16<00:21,  5.27s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 4 / 3 / 26:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:16<00:21,  5.27s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 4 / 3 / 26:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:23<00:15,  5.30s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 4 / 3 / 27:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:23<00:15,  5.30s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 4 / 3 / 27:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:34<00:11,  5.52s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 3 / 28:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:34<00:11,  5.52s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 3 / 28:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:37<00:05,  5.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 5 / 3 / 29:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:37<00:05,  5.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 5 / 3 / 29: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:41<00:00,  5.38s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 5 / 3 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:41<00:00,  5.38s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628631185.ta.chkpt\" at 2024-04-20 15:57:11 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 5 / 3 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:41<00:00,  5.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 22     |\n",
      "| Number of failed attacks:     | 5      |\n",
      "| Number of skipped attacks:    | 3      |\n",
      "| Original accuracy:            | 90.0%  |\n",
      "| Accuracy under attack:        | 16.67% |\n",
      "| Attack success rate:          | 81.48% |\n",
      "| Average perturbed word %:     | 10.57% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 366.0  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_41.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                                                                    | 0/30 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|████▏                                                                                                                       | 1/30 [00:02<01:20,  2.76s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 0 / 1:   3%|██▎                                                                   | 1/30 [00:02<01:20,  2.78s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 1 / 2:   7%|████▋                                                                 | 2/30 [00:02<00:39,  1.41s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 1 / 2:  10%|███████                                                               | 3/30 [00:06<00:58,  2.16s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 1 / 3:  10%|███████                                                               | 3/30 [00:06<00:58,  2.17s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 2 / 4:  13%|█████████▎                                                            | 4/30 [00:06<00:42,  1.63s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 2 / 4:  17%|███████████▋                                                          | 5/30 [00:09<00:49,  1.99s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 3 / 0 / 2 / 5:  17%|███████████▋                                                          | 5/30 [00:09<00:49,  1.99s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628648549.ta.chkpt\" at 2024-04-20 15:57:28 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 2 / 6:  20%|██████████████                                                        | 6/30 [00:13<00:54,  2.25s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 2 / 6:  23%|████████████████▎                                                     | 7/30 [00:17<00:56,  2.46s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 2 / 7:  23%|████████████████▎                                                     | 7/30 [00:17<00:56,  2.47s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 2 / 8:  27%|██████████████████▋                                                   | 8/30 [00:18<00:49,  2.25s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 2 / 8:  30%|█████████████████████                                                 | 9/30 [00:22<00:52,  2.48s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 1 / 2 / 9:  30%|█████████████████████                                                 | 9/30 [00:22<00:52,  2.48s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 2 / 10:  33%|██████████████████████▋                                             | 10/30 [00:25<00:50,  2.54s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628663954.ta.chkpt\" at 2024-04-20 15:57:43 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 2 / 10:  37%|████████████████████████▉                                           | 11/30 [00:29<00:50,  2.65s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 1 / 2 / 11:  37%|████████████████████████▉                                           | 11/30 [00:29<00:50,  2.66s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 2 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:34<00:51,  2.87s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 2 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:42<01:03,  3.50s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 2 / 12:  43%|█████████████████████████████▍                                      | 13/30 [00:42<00:55,  3.28s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 2 / 13:  43%|█████████████████████████████▍                                      | 13/30 [00:42<00:55,  3.28s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 2 / 13:  47%|███████████████████████████████▋                                    | 14/30 [00:46<00:53,  3.33s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 2 / 2 / 14:  47%|███████████████████████████████▎                                   | 14/30 [00:46<00:53,  3.33s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 2 / 2 / 14:  50%|█████████████████████████████████▌                                 | 15/30 [00:55<00:55,  3.68s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 2 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [00:55<00:55,  3.68s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628693770.ta.chkpt\" at 2024-04-20 15:58:13 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 2 / 15:  53%|███████████████████████████████████▋                               | 16/30 [01:08<00:59,  4.28s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 4 / 2 / 16:  53%|███████████████████████████████████▋                               | 16/30 [01:08<00:59,  4.28s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 4 / 2 / 16:  57%|█████████████████████████████████████▉                             | 17/30 [01:11<00:54,  4.23s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 2 / 17:  57%|█████████████████████████████████████▉                             | 17/30 [01:11<00:55,  4.23s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 2 / 17:  60%|████████████████████████████████████████▏                          | 18/30 [01:15<00:50,  4.17s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 4 / 2 / 18:  60%|████████████████████████████████████████▏                          | 18/30 [01:15<00:50,  4.17s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 4 / 2 / 18:  63%|██████████████████████████████████████████▍                        | 19/30 [01:21<00:47,  4.31s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 4 / 2 / 19:  63%|██████████████████████████████████████████▍                        | 19/30 [01:21<00:47,  4.31s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 4 / 2 / 19:  67%|████████████████████████████████████████████▋                      | 20/30 [01:27<00:43,  4.35s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 4 / 2 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:27<00:43,  4.35s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628725626.ta.chkpt\" at 2024-04-20 15:58:45 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 4 / 2 / 20:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:32<00:39,  4.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 4 / 2 / 21:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:32<00:39,  4.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 4 / 2 / 21:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:41<00:37,  4.63s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 2 / 22:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:41<00:37,  4.63s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 2 / 22:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:47<00:32,  4.67s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 5 / 2 / 23:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:47<00:32,  4.67s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 5 / 2 / 23:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:54<00:28,  4.79s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 5 / 2 / 24:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:54<00:28,  4.79s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 5 / 2 / 24:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:08<00:25,  5.13s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 5 / 2 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:08<00:25,  5.13s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628766967.ta.chkpt\" at 2024-04-20 15:59:26 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 5 / 2 / 25:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:19<00:21,  5.35s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 5 / 2 / 26:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:19<00:21,  5.35s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 5 / 2 / 26:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:25<00:16,  5.38s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 2 / 27:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:25<00:16,  5.38s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 2 / 27:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:36<00:11,  5.58s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 6 / 2 / 28:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:36<00:11,  5.58s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 6 / 2 / 28:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:39<00:05,  5.49s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 6 / 2 / 29:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:39<00:05,  5.49s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 6 / 2 / 29: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:43<00:00,  5.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 6 / 2 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:43<00:00,  5.44s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628801729.ta.chkpt\" at 2024-04-20 16:00:01 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 6 / 2 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:43<00:00,  5.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 22     |\n",
      "| Number of failed attacks:     | 6      |\n",
      "| Number of skipped attacks:    | 2      |\n",
      "| Original accuracy:            | 93.33% |\n",
      "| Accuracy under attack:        | 20.0%  |\n",
      "| Attack success rate:          | 78.57% |\n",
      "| Average perturbed word %:     | 8.44%  |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 359.39 |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_46.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                                                                    | 0/30 [00:00<?, ?it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 1 / 1:   3%|██▎                                                                   | 1/30 [00:00<00:01, 14.99it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 2 / 2:   7%|████▋                                                                 | 2/30 [00:00<00:01, 17.12it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 2 / 2:  10%|███████                                                               | 3/30 [00:03<00:34,  1.28s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 2 / 3:  10%|███████                                                               | 3/30 [00:03<00:34,  1.29s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 3 / 4:  13%|█████████▎                                                            | 4/30 [00:03<00:25,  1.03it/s]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 3 / 5:  17%|███████████▋                                                          | 5/30 [00:07<00:36,  1.46s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628818686.ta.chkpt\" at 2024-04-20 16:00:18 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 3 / 5:  20%|██████████████                                                        | 6/30 [00:10<00:42,  1.79s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 3 / 0 / 3 / 6:  20%|██████████████                                                        | 6/30 [00:10<00:43,  1.79s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 3 / 7:  23%|████████████████▎                                                     | 7/30 [00:14<00:47,  2.06s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 1 / 3 / 8:  27%|██████████████████▋                                                   | 8/30 [00:15<00:41,  1.90s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 1 / 3 / 8:  30%|█████████████████████                                                 | 9/30 [00:22<00:51,  2.47s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 3 / 9:  30%|█████████████████████                                                 | 9/30 [00:22<00:51,  2.47s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 1 / 3 / 9:  33%|███████████████████████                                              | 10/30 [00:25<00:50,  2.52s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 1 / 3 / 10:  33%|██████████████████████▋                                             | 10/30 [00:25<00:50,  2.52s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628836581.ta.chkpt\" at 2024-04-20 16:00:36 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 1 / 3 / 10:  37%|████████████████████████▉                                           | 11/30 [00:29<00:50,  2.65s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 3 / 11:  37%|████████████████████████▉                                           | 11/30 [00:29<00:50,  2.65s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 3 / 11:  40%|███████████████████████████▏                                        | 12/30 [00:37<00:56,  3.17s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 2 / 3 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:38<00:57,  3.17s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 2 / 3 / 12:  43%|█████████████████████████████▍                                      | 13/30 [00:46<01:00,  3.57s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 3 / 3 / 13:  43%|█████████████████████████████▍                                      | 13/30 [00:46<01:00,  3.57s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 3 / 3 / 13:  47%|███████████████████████████████▋                                    | 14/30 [00:49<00:57,  3.57s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 3 / 3 / 14:  47%|███████████████████████████████▋                                    | 14/30 [00:49<00:57,  3.57s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 3 / 3 / 14:  50%|██████████████████████████████████                                  | 15/30 [00:58<00:58,  3.91s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 4 / 3 / 15:  50%|██████████████████████████████████                                  | 15/30 [00:58<00:58,  3.91s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628870092.ta.chkpt\" at 2024-04-20 16:01:10 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 4 / 3 / 15:  53%|████████████████████████████████████▎                               | 16/30 [01:11<01:02,  4.48s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 4 / 3 / 16:  53%|████████████████████████████████████▎                               | 16/30 [01:11<01:02,  4.48s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 4 / 3 / 16:  57%|██████████████████████████████████████▌                             | 17/30 [01:15<00:57,  4.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 4 / 3 / 17:  57%|█████████████████████████████████████▉                             | 17/30 [01:15<00:57,  4.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 4 / 4 / 18:  60%|████████████████████████████████████████▏                          | 18/30 [01:15<00:50,  4.19s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 4 / 4 / 18:  63%|██████████████████████████████████████████▍                        | 19/30 [01:22<00:47,  4.34s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 4 / 19:  63%|██████████████████████████████████████████▍                        | 19/30 [01:22<00:47,  4.34s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 4 / 4 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:27<00:43,  4.37s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628898725.ta.chkpt\" at 2024-04-20 16:01:38 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 4 / 4 / 20:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:33<00:39,  4.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 4 / 4 / 21:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:33<00:39,  4.43s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 4 / 4 / 21:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:42<00:37,  4.66s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 5 / 4 / 22:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:42<00:37,  4.66s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 5 / 4 / 22:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:47<00:32,  4.69s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 5 / 4 / 23:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:47<00:32,  4.69s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 5 / 4 / 23:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:55<00:28,  4.82s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 4 / 24:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:55<00:28,  4.83s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 4 / 24:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:09<00:25,  5.18s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 5 / 4 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:09<00:25,  5.18s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628940866.ta.chkpt\" at 2024-04-20 16:02:20 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 5 / 4 / 25:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:25<00:22,  5.60s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 6 / 4 / 26:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:25<00:22,  5.60s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 6 / 4 / 26:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:31<00:16,  5.62s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 6 / 4 / 27:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:31<00:16,  5.62s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 6 / 4 / 27:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:42<00:11,  5.81s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 7 / 4 / 28:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:42<00:11,  5.81s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 7 / 4 / 28:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:45<00:05,  5.71s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 7 / 4 / 29:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:45<00:05,  5.71s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 7 / 4 / 29: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:49<00:00,  5.65s/it]\u001b[A\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 7 / 4 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:49<00:00,  5.65s/it]\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713628980885.ta.chkpt\" at 2024-04-20 16:03:00 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 7 / 4 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [02:49<00:00,  5.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 19     |\n",
      "| Number of failed attacks:     | 7      |\n",
      "| Number of skipped attacks:    | 4      |\n",
      "| Original accuracy:            | 86.67% |\n",
      "| Accuracy under attack:        | 23.33% |\n",
      "| Attack success rate:          | 73.08% |\n",
      "| Average perturbed word %:     | 9.85%  |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 399.65 |\n",
      "+-------------------------------+--------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for rank in range(1, 50, 5):\n",
    "    model.load_state_dict(torch.load(\"weight_trained_1.pth\"))\n",
    "    \n",
    "    all_model_weight = svd_sa(model, rank)\n",
    "    model.load_state_dict(all_model_weight)\n",
    "    \n",
    "    model_wrapper = PyTorchModelWrapper(model, tokenizer)\n",
    "    goal_function = TargetedClassification(model_wrapper, target_class = 1)\n",
    "    attack = Attack(goal_function, constraints, transformation, search_method)\n",
    "    \n",
    "    attack_args = textattack.AttackArgs(\n",
    "    num_examples=30,\n",
    "    log_to_csv=\"rank/log_\"+str(rank)+\".csv\",\n",
    "    checkpoint_interval=5,\n",
    "    checkpoint_dir=\"checkpoints\",\n",
    "    disable_stdout=True\n",
    "    )\n",
    "    \n",
    "    attacker = Attacker(attack, dataset, attack_args)\n",
    "    attacker.attack_dataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "785b8b40-7250-4898-b64a-4e2e3b2062ad",
   "metadata": {},
   "source": [
    "### Result Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "862bd78a-444a-4c6e-8938-a33c176a9965",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "9a2d92cd-44b9-4f29-8785-8aeaf8de8356",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_df = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "97502de0-1dc6-4da2-ac37-309e614f0465",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for i in range(1, 50, 5):\n",
    "    df = pd.read_csv(\"rank/log_\" + str(i) + \".csv\")\n",
    "    all_df.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e60e283c-b27b-4b23-90ef-898f8643f489",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     Failed\n",
       "1     Failed\n",
       "2     Failed\n",
       "3     Failed\n",
       "4     Failed\n",
       "5     Failed\n",
       "6     Failed\n",
       "7     Failed\n",
       "8     Failed\n",
       "9     Failed\n",
       "10    Failed\n",
       "11    Failed\n",
       "12    Failed\n",
       "13    Failed\n",
       "14    Failed\n",
       "15    Failed\n",
       "16    Failed\n",
       "17    Failed\n",
       "18    Failed\n",
       "19    Failed\n",
       "20    Failed\n",
       "21    Failed\n",
       "22    Failed\n",
       "23    Failed\n",
       "24    Failed\n",
       "25    Failed\n",
       "26    Failed\n",
       "27    Failed\n",
       "28    Failed\n",
       "29    Failed\n",
       "Name: result_type, dtype: object"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df[0][\"result_type\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36614343-8945-40e3-acd2-d5c0dcca1774",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "88c27e1e-a9c1-4979-af69-bfd362996d34",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "success = []\n",
    "failed = []\n",
    "skipped = []\n",
    "for i in range(len(all_df)):\n",
    "    try:\n",
    "        success.append(all_df[i][\"result_type\"].value_counts()[\"Successful\"])\n",
    "    except:\n",
    "        success.append(0)\n",
    "    failed.append(all_df[i][\"result_type\"].value_counts()[\"Failed\"])\n",
    "    try:\n",
    "        skipped.append(all_df[i][\"result_type\"].value_counts()[\"Skipped\"])\n",
    "    except:\n",
    "        skipped.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "f70ea510-0fbc-41ab-accb-18a64bdd02ba",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 20, 20, 23, 20, 21, 21, 22, 22, 19]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d685db2c-728b-4893-8851-a75bb2217348",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "e6077336-e23d-4163-8084-ffdb805ad5ff",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAu0AAAHHCAYAAAALYWaqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/OQEPoAAAACXBIWXMAAA9hAAAPYQGoP6dpAABT+klEQVR4nO3deVyVZf7/8feRXRFwR5IEA8UFTXHN3BdEtFTMJU00dcZcSs2mcRoX2rRyaSyzZkLRBnNLrNRs3EAz98Kl3MO03NIEBAMS7t8f/jzfTiyCgeeWXs/H4zwenfu67uv+3Pc55JuL69zHYhiGIQAAAACmVcbeBQAAAAAoGKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEduMtOnz4ti8WiWbNm2buUuyomJkYWi0WnT5+22f7GG2+oVq1acnBw0IMPPmiX2u62+Ph4WSwWxcfH27sU0/Dz89PQoUPtXQYAmBahHaXOO++8I4vFohYtWuTZ/u2332r69Om5wuOtfWNiYkq2wHvETz/9pGeeeUZBQUFyc3NT1apV1bx5cz3//PNKS0srlmP873//09/+9je1bt1aixYt0quvvlqo/fr16yeLxaLnn38+z/b169dr+vTpubZfv35d06dPv2th2Yzvp/bt28tisSgwMDDP9o0bN8pischisWjVqlVFHr+gny8AwJ0jtKPUiY2NlZ+fn/bs2aOTJ0/mav/2228VFRVFaC/Azz//rKZNm2rJkiUKDw/XvHnzNHHiRAUEBGjBggW6fPlysRxny5YtKlOmjKKjozVkyBB17979tvukpqbq008/lZ+fnz788EMZhpGrz/r16xUVFZVr+/Xr1xUVFWX30N62bVv98ssvatu27V2p4/dcXV118uRJ7dmzJ1dbbGysXF1d73jsgn6+CnLs2DH95z//uePjAkBp52jvAoDilJSUpC+//FKrV6/WX//6V8XGxmratGn2LuueEx0drTNnzmjHjh166KGHbNpSU1Pl7OxcLMe5dOmS3NzcijTeRx99pOzsbC1cuFAdO3bUtm3b1K5du2Kp524pU6bMHwrGf9QDDzygGzdu6MMPP1Tz5s2t2zMyMhQXF6fw8HB99NFHJV6HYRjKyMiQm5ubXFxcSvx4AHAvY6YdpUpsbKwqVKig8PBw9e3bV7GxsTbtMTExeuyxxyRJHTp0sC4DiI+Pl5+fn7755hslJCRYt7dv317SzZnnSZMmKTg4WO7u7vLw8FBYWJgOHDiQq4aMjAxNnz5dtWvXlqurq6pXr64+ffro1KlT+dZtGIb+8pe/yNnZWatXry7wHNPT0/Xss8/K19dXLi4uqlOnjmbNmpVrxtlisWjs2LFas2aNGjRoIBcXF9WvX18bNmy47XU8deqUHBwc1LJly1xtHh4euQLn7t271a1bN3l6eqps2bJq166dduzYUeAxLBaLFi1apPT0dOv1LsxfOWJjY9WlSxd16NBBdevWzfUaDx06VPPnz7ce49bj9OnTqlKliiQpKirKuv23y2iOHj2qvn37qmLFinJ1dVXTpk31ySef2Ix/a23+jh07NHHiRFWpUkXlypVT79699dNPP1n7FfR+ym9N+8qVKxUSEiI3NzdVrlxZgwcP1o8//pjr/Nzd3fXjjz+qV69ecnd3V5UqVTRp0iRlZ2ff9vrdMnDgQC1fvlw5OTnWbZ9++qmuX7+ufv365er//fffa/To0apTp47c3NxUqVIlPfbYYzYz6gX9fN26Jj169NDnn3+upk2bys3NTe+995617daadsMw1KFDB1WpUkWXLl2yjp+VlaXg4GA98MADSk9PL/S5AkBpQGhHqRIbG6s+ffrI2dlZAwcO1IkTJ7R3715re9u2bfX0009Lkv7xj3/ogw8+0AcffKC6devqzTffVI0aNRQUFGTd/sILL0iSvvvuO61Zs0Y9evTQnDlz9Nxzz+nQoUNq166dzp07Zx0/OztbPXr0UFRUlEJCQjR79mw988wzSklJ0eHDh/OsOTs7W0OHDtWSJUsUFxenPn365Ht+hmHokUce0dy5c9WtWzfNmTNHderU0XPPPaeJEyfm6v/FF19o9OjRGjBggF5//XVlZGQoIiJCV65cKfA61qxZU9nZ2frggw8K7CfdXOLStm1bpaamatq0aXr11VeVnJysjh075rn84pYPPvhAbdq0kYuLi/V63265yLlz57R161YNHDhQ0s3guWrVKmVlZVn7/PWvf1WXLl2sx7j1qFKlihYsWCBJ6t27t3X7rev9zTffqGXLljpy5Ij+/ve/a/bs2SpXrpx69eqluLi4XLWMGzdOBw4c0LRp0/TUU0/p008/1dixY63tBb2f8hITE6N+/frJwcFBM2bM0MiRI7V69Wo9/PDDSk5OtumbnZ2t0NBQVapUSbNmzVK7du00e/Zs/fvf/y7w+v3W448/rvPnz9v84rB06VJ16tRJVatWzdV/7969+vLLLzVgwADNmzdPo0aN0ubNm9W+fXtdv35dUsE/X7ccO3ZMAwcOVJcuXfSvf/0rzw8fWywWLVy4UBkZGRo1apR1+7Rp0/TNN99o0aJFKleuXKHPFQBKBQMoJfbt22dIMjZu3GgYhmHk5OQYNWrUMJ555hmbfitXrjQkGVu3bs01Rv369Y127drl2p6RkWFkZ2fbbEtKSjJcXFyMF1980bpt4cKFhiRjzpw5ucbIycmx7ifJeOONN4xff/3V6N+/v+Hm5mZ8/vnntz3HNWvWGJKMl19+2WZ73759DYvFYpw8edK6TZLh7Oxss+3AgQOGJOOtt94q8DgXLlwwqlSpYkgygoKCjFGjRhlLly41kpOTc51TYGCgERoaaj0/wzCM69evG/7+/kaXLl2s2xYtWmRIMpKSkqzbIiMjjXLlyt32vG+ZNWuW4ebmZqSmphqGYRjHjx83JBlxcXE2/caMGWPk9b+3n376yZBkTJs2LVdbp06djODgYCMjI8Pm/B566CEjMDAw13l07tzZ5pwnTJhgODg42Fyj/N5PW7dutXkPZmVlGVWrVjUaNGhg/PLLL9Z+a9euNSQZU6dOtW6LjIw0JNm87wzDMBo3bmyEhITkOtbvtWvXzqhfv75hGIbRtGlTY/jw4YZhGMbVq1cNZ2dnY/Hixdb6Vq5cad3v+vXrucbauXOnIclYsmSJdVtBP181a9Y0JBkbNmzIsy0yMtJm23vvvWdIMv773/8au3btMhwcHIzx48ff9hwBoDRiph2lRmxsrKpVq6YOHTpIujlb179/fy1btqxIywby4uLiojJlbv64ZGdn68qVK3J3d1edOnX01VdfWft99NFHqly5ssaNG5drDIvFYvM8KytLjz32mNauXav169era9eut61j/fr1cnBwsM5m3vLss8/KMAx99tlnNts7d+6sBx54wPq8YcOG8vDw0HfffVfgcapVq6YDBw5o1KhRunr1qt599109/vjjqlq1ql566SXrUpzExESdOHFCjz/+uK5cuaLLly/r8uXLSk9PV6dOnbRt2zab5Rd/VGxsrMLDw1W+fHlJUmBgoEJCQnItkSmqn3/+WVu2bFG/fv107do163lcuXJFoaGhOnHiRK5lKn/5y19sXtM2bdooOztb33//fZGPv2/fPl26dEmjR4+2WXoUHh6uoKAgrVu3Ltc+v52BvnX8272uv/f4449r9erVysrK0qpVq+Tg4KDevXvn2dfNzc3637/++quuXLmigIAAeXl52fwM3I6/v79CQ0ML1fcvf/mLQkNDNW7cOD3xxBN64IEHCn2HIQAobQjtKBWys7O1bNkydejQQUlJSTp58qROnjypFi1a6OLFi9q8efMfGj8nJ0dz585VYGCgXFxcVLlyZVWpUkUHDx5USkqKtd+pU6dUp04dOTre/jPeM2bM0Jo1a7Rq1SrrWufb+f777+Xj42MNrbfcWn7w+8B4//335xqjQoUKunr16m2PVb16dS1YsEDnz5/XsWPHNG/ePFWpUkVTp05VdHS0JOnEiROSpMjISFWpUsXm8f777yszM9Pm+vwRR44c0ddff63WrVtbX9+TJ0+qffv2Wrt2rVJTU+947JMnT8owDE2ZMiXXedz6IPNv11ZLua9thQoVJKlQ1/b3br1uderUydUWFBSU63V1dXW1rs//7fGLeuwBAwYoJSVFn332mWJjY9WjR49c761bfvnlF02dOtX6WYpbPwPJyclFeo39/f2LVGN0dLSuX7+uEydOKCYmxuaXBwD4M+HuMSgVtmzZovPnz2vZsmVatmxZrvbY2NhCzWTn59VXX9WUKVP05JNP6qWXXlLFihVVpkwZjR8//o5nkkNDQ7Vhwwa9/vrrat++fYncTcTBwSHP7UYet0nMj8ViUe3atVW7dm2Fh4crMDBQsbGxGjFihPXc33jjjXy/GMnd3b3Ideflv//9ryRpwoQJmjBhQq72jz76SMOGDbujsW+dx6RJk/KdBQ4ICLB5XhzX9k7ld+yiql69utq3b6/Zs2drx44dBd4xZty4cVq0aJHGjx+vVq1aydPTUxaLRQMGDCjSz0BRQ3d8fLwyMzMlSYcOHVKrVq2KtD8AlBaEdpQKsbGxqlq1qvWuIb+1evVqxcXF6d1335Wbm1uuZSq/lV/bqlWr1KFDB+sM8y3JycmqXLmy9fkDDzyg3bt369dff5WTk1OBNbds2VKjRo1Sjx499NhjjykuLu62M/Q1a9bUpk2bdO3aNZsZ0aNHj1rbS1KtWrVUoUIFnT9/XpKsS288PDzUuXPnEjuuYRhaunSpOnTooNGjR+dqf+mllxQbG2sN7fm9jvltr1WrliTJycmpWM+joPfab9163Y4dO6aOHTvatB07dqxEX9fHH39cI0aMkJeXV4H3yV+1apUiIyM1e/Zs67aMjIxcH5It7DkXxvnz5zVu3Dh17dpVzs7O1l+qSvp9DgBmxPIY3PN++eUXrV69Wj169FDfvn1zPcaOHatr165Zb913664Tvw8bt9ry2u7g4JBrBnXlypW51jlHRETo8uXLevvtt3ONkdcMbOfOnbVs2TJt2LBBTzzxxG1nLLt3767s7Oxc48+dO1cWi0VhYWEF7l9Yu3fvzvOWenv27NGVK1esyzhCQkL0wAMPaNasWXl+S+pvb4H4R+zYsUOnT5/WsGHD8nyN+/fvr61bt1rv5JPfa1y2bNk8t1etWlXt27fXe++9Z/2FpDjOI7/30+81bdpUVatW1bvvvmudVZakzz77TEeOHFF4ePgdHb8w+vbtq2nTpumdd94p8H75ef0MvPXWW7k+L1LQz1dRjRw5Ujk5OYqOjta///1vOTo6avjw4XflrxkAYDbMtOOe98knn+jatWt65JFH8mxv2bKlqlSpotjYWPXv318PPvigHBwc9NprryklJUUuLi7q2LGjqlatqpCQEC1YsEAvv/yyAgICVLVqVXXs2FE9evTQiy++qGHDhumhhx7SoUOHFBsba52hvWXIkCFasmSJJk6cqD179qhNmzZKT0/Xpk2bNHr0aD366KO56uvVq5cWLVqkIUOGyMPDw3rf6rz07NlTHTp00AsvvKDTp0+rUaNG+t///qePP/5Y48ePt/nQ6R/xwQcfKDY2Vr1791ZISIicnZ115MgRLVy4UK6urvrHP/4h6eaXBL3//vsKCwtT/fr1NWzYMN1333368ccftXXrVnl4eOjTTz/9w/XExsbKwcEh3/D6yCOP6IUXXtCyZcs0ceJEhYSESJKefvpphYaGysHBQQMGDJCbm5vq1aun5cuXq3bt2qpYsaIaNGigBg0aaP78+Xr44YcVHByskSNHqlatWrp48aJ27typH374Ic978t9Ofu+n33NyctJrr72mYcOGqV27dho4cKAuXryof/3rX/Lz88tzOVBx8fT0tLlXfX569OihDz74QJ6enqpXr5527typTZs2qVKlSjb9Cvr5KopFixZp3bp1iomJUY0aNSTd/CVh8ODBWrBgQZ5/cQGAUs1et60BikvPnj0NV1dXIz09Pd8+Q4cONZycnIzLly8bhmEY//nPf4xatWoZDg4ONrenu3DhghEeHm6UL1/ekGS9XV9GRobx7LPPGtWrVzfc3NyM1q1bGzt37jTatWuX65Z+169fN1544QXD39/fcHJyMry9vY2+ffsap06dMgzD9paPv/XOO+8YkoxJkyYVeL7Xrl0zJkyYYPj4+BhOTk5GYGCg8cYbb9jcftAwbt7yccyYMbn2z+vWer938OBB47nnnjOaNGliVKxY0XB0dDSqV69uPPbYY8ZXX32Vq//XX39t9OnTx6hUqZLh4uJi1KxZ0+jXr5+xefNma587veVjVlaWUalSJaNNmzYF9vP39zcaN25sGIZh3Lhxwxg3bpxRpUoVw2Kx2Nz+8csvvzRCQkIMZ2fnXLd/PHXqlDFkyBDD29vbcHJyMu677z6jR48exqpVq3Kdx969e22O//vbOBpG/u+nvPoahmEsX77caNy4seHi4mJUrFjRGDRokPHDDz/Y9Mnvmk2bNi3P21z+3m9v+ZifvG75ePXqVWPYsGFG5cqVDXd3dyM0NNQ4evRonu+n/H6+atasaYSHh+d5zN+Oc/bsWcPT09Po2bNnrn69e/c2ypUrZ3z33Xe3PVcAKE0shsHfGQEAAAAzY007AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATK7Uf7lSTk6Ozp07p/Llyxfr12sDAICSYxiGrl27Jh8fH5UpwxwjUOpD+7lz5+Tr62vvMgAAwB04e/as9VtxgT+zUh/ay5cvL+nmD72Hh4edqwEAAIWRmpoqX19f67/jwJ9dqQ/tt5bEeHh4ENoBALjHsLQVuIlFYgAAAIDJEdoBAAAAkyO0AwAAACZX6te0AwAA3KmcnBxlZWXZuwyUUk5OTnJwcChUX0I7AABAHrKyspSUlKScnBx7l4JSzMvLS97e3rf90DWhHQAA4HcMw9D58+fl4OAgX19fvuAJxc4wDF2/fl2XLl2SJFWvXr3A/oR2AACA37lx44auX78uHx8flS1b1t7loJRyc3OTJF26dElVq1YtcKkMvzYCAAD8TnZ2tiTJ2dnZzpWgtLv1S+Gvv/5aYD9COwAAQD74cieUtMK+xwjtAAAAgMnZNbQvWLBADRs2lIeHhzw8PNSqVSt99tln1vaMjAyNGTNGlSpVkru7uyIiInTx4kU7VgwAAIDCio+Pl8ViUXJysr1LuefZ9YOoNWrU0MyZMxUYGCjDMLR48WI9+uij+vrrr1W/fn1NmDBB69at08qVK+Xp6amxY8eqT58+2rFjhz3LBgAAf1J+f193V493emb4He23c+dOPfzww+rWrZvWrbOtefr06VqzZo0SExNttlssFsXFxalXr153WG3x+OmnnzR16lStW7dOFy9eVIUKFdSoUSNNnTpVrVu3tmtt9mTX0N6zZ0+b56+88ooWLFigXbt2qUaNGoqOjtbSpUvVsWNHSdKiRYtUt25d7dq1Sy1btrRHyQAAAKYXHR2tcePGKTo6WufOnZOPj4+9Syq0iIgIZWVlafHixapVq5YuXryozZs368qVK/Yuza5Ms6Y9Oztby5YtU3p6ulq1aqX9+/fr119/VefOna19goKCdP/992vnzp12rBQAAMC80tLStHz5cj311FMKDw9XTEyMtS0mJkZRUVE6cOCALBaLLBaLYmJi5OfnJ0nq3bu3LBaL9fmpU6f06KOPqlq1anJ3d1ezZs20adMmm+NlZmbq+eefl6+vr1xcXBQQEKDo6Og8a7t+/brCwsLUunXrPJfMJCcna/v27XrttdfUoUMH1axZU82bN9fkyZP1yCOPSJJOnz4ti8Vi85eC5ORkWSwWxcfHW7d988036tGjhzw8PFS+fHm1adNGp06dsrYvXLhQ9evXl4uLi6pXr66xY8fajDdixAhVqVJFHh4e6tixow4cOGBtP3DggDp06KDy5cvLw8NDISEh2rdvnyTp+++/V8+ePVWhQgWVK1dO9evX1/r16/N9vQrL7vdpP3TokFq1aqWMjAy5u7srLi5O9erVU2JiopydneXl5WXTv1q1arpw4UK+42VmZiozM9P6PDU1taRKBwAAMJ0VK1YoKChIderU0eDBgzV+/HhNnjxZFotF/fv31+HDh7VhwwZr+Pb09FR4eLiqVq2qRYsWqVu3btb7haelpal79+565ZVX5OLioiVLlqhnz546duyY7r//fknSkCFDtHPnTs2bN0+NGjVSUlKSLl++nKuu5ORkhYeHy93dXRs3bszz/vfu7u5yd3fXmjVr1LJlS7m4uNzRNfjxxx/Vtm1btW/fXlu2bJGHh4d27NihGzduSLr5ucqJEydq5syZCgsLU0pKis3y68cee0xubm767LPP5Onpqffee0+dOnXS8ePHVbFiRQ0aNEiNGzfWggUL5ODgoMTERDk5OUmSxowZo6ysLG3btk3lypXTt99+K3d39zs6j9+ye2ivU6eOEhMTlZKSolWrVikyMlIJCQl3PN6MGTMUFRVVjBUWg+meheiTUvJ1AACAUi86OlqDBw+WJHXr1k0pKSlKSEhQ+/bt5ebmJnd3dzk6Osrb29u6z60v+fHy8rLZ3qhRIzVq1Mj6/KWXXlJcXJw++eQTjR07VsePH9eKFSu0ceNG6+qIWrVq5arpwoUL6t+/vwIDA7V06dJ873/v6OiomJgYjRw5Uu+++66aNGmidu3aacCAAWrYsGGhr8H8+fPl6empZcuWWcN07dq1re0vv/yynn32WT3zzDPWbc2aNZMkffHFF9qzZ48uXbpk/aVh1qxZWrNmjVatWqW//OUvOnPmjJ577jkFBQVJkgIDA63jnDlzRhEREQoODs73etwJuy+PcXZ2VkBAgEJCQjRjxgw1atRI//rXv+Tt7a2srKxcfzq5ePGizZvp9yZPnqyUlBTr4+zZsyV8BgAAAOZw7Ngx7dmzRwMHDpR0MwT3798/3+Uqt5OWlqZJkyapbt268vLykru7u44cOaIzZ85IkhITE+Xg4KB27doVOE6XLl0UEBCg5cuX3/YLqyIiInTu3Dl98skn6tatm+Lj49WkSRObZT63k5iYqDZt2lgD+29dunRJ586dU6dOnfLc98CBA0pLS7PevfDWIykpybq8ZuLEiRoxYoQ6d+6smTNn2iy7efrpp/Xyyy+rdevWmjZtmg4ePFjougti99D+ezk5OcrMzFRISIicnJy0efNma9uxY8d05swZtWrVKt/9XVxcrLeQvPUAAAD4M4iOjtaNGzfk4+MjR0dHOTo6asGCBfroo4+UklL0v+pPmjRJcXFxevXVV7V9+3YlJiYqODhYWVlZkv5vhv52wsPDtW3bNn377beF6u/q6qouXbpoypQp+vLLLzV06FBNmzZNklSmzM34ahiGtf/vv020oLpuV3NaWpqqV6+uxMREm8exY8f03HPPSbp5B55vvvlG4eHh2rJli+rVq6e4uDhJ0ogRI/Tdd9/piSee0KFDh9S0aVO99dZbhTrvgtg1tE+ePFnbtm3T6dOndejQIU2ePFnx8fEaNGiQPD09NXz4cE2cOFFbt27V/v37NWzYMLVq1Yo7xwAAAPzOjRs3tGTJEs2ePdsmbB44cEA+Pj768MMPJd1c5ZCdnZ1rfycnp1zbd+zYoaFDh6p3794KDg6Wt7e3Tp8+bW0PDg5WTk7ObZc2z5w5U5GRkerUqVOhg/tv1atXT+np6ZKkKlWqSJLOnz9vbf/97SsbNmyo7du35wrzklS+fHn5+fnZTAz/VpMmTXThwgU5OjoqICDA5lG5cmVrv9q1a2vChAn63//+pz59+mjRokXWNl9fX40aNUqrV6/Ws88+q//85z9FPuffs+ua9kuXLmnIkCE6f/68PD091bBhQ33++efq0qWLJGnu3LkqU6aMIiIilJmZqdDQUL3zzjv2LBkAAMCU1q5dq6tXr2r48OHy9LT9PF1ERISio6M1atQo+fn5KSkpSYmJiapRo4bKly8vFxcXa5Bt3bq1XFxcVKFCBQUGBmr16tXq2bOnLBaLpkyZopycHOu4fn5+ioyM1JNPPmn9IOr333+vS5cuqV+/fjY1zJo1S9nZ2erYsaPi4+Ot68F/68qVK3rsscf05JNPqmHDhipfvrz27dun119/XY8++qikmzPlLVu21MyZM+Xv769Lly7pn//8p804Y8eO1VtvvaUBAwZo8uTJ8vT01K5du9S8eXPVqVNH06dP16hRo1S1alWFhYXp2rVr2rFjh8aNG6fOnTurVatW6tWrl15//XXVrl1b586d07p169S7d2/Vr19fzz33nPr27St/f3/98MMP2rt3ryIiIiRJ48ePV1hYmGrXrq2rV69q69atqlu37h9+fe060x4dHa3Tp08rMzNTly5d0qZNm6yBXbr5p5H58+fr559/Vnp6ulavXl3genYAAIA/q+joaHXu3DlXYJduhvZ9+/bp4MGDioiIULdu3dShQwdVqVLFOgM/e/Zsbdy4Ub6+vmrcuLEkac6cOapQoYIeeugh9ezZU6GhoWrSpInN2AsWLFDfvn01evRoBQUFaeTIkdZZ8d+bO3eu+vXrp44dO+r48eO52t3d3dWiRQvNnTtXbdu2VYMGDTRlyhSNHDlSb7/9trXfwoULdePGDYWEhGj8+PF6+eWXbcapVKmStmzZorS0NLVr104hISH6z3/+Y13jHhkZqTfffFPvvPOO6tevrx49eujEiROSbn7J1Pr169W2bVsNGzZMtWvX1oABA/T999+rWrVqcnBw0JUrVzRkyBDVrl1b/fr1U1hYmPVGKNnZ2RozZozq1q2rbt26qXbt2sUy6WwxfrsgqBRKTU2Vp6enUlJS7Le+nbvHAABQJPb+9zsjI0NJSUny9/eXq6vrXT8+/jwK+14z3QdRAQAAANgitAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAKBExMfHy2KxKDk5+a4cLyYmRl5eXtbn06dP14MPPviHxjx9+rQsFosSExP/0Dh/lKNdjw4AAHAvme55l4+Xcke77dy5Uw8//LC6deumdevW2Q45fbrWrFmTK4RaLBbFxcWpV69ed1hs8Rg6dKgWL16ca/uJEycUEBBQ4L79+/dX9+7dS6o0u2KmHQAAoJSJjo7WuHHjtG3bNp07d87e5RRZt27ddP78eZuHv7//bfdzc3NT1apV70KFdx+hHQAAoBRJS0vT8uXL9dRTTyk8PFwxMTHWtpiYGEVFRenAgQOyWCyyWCyKiYmRn5+fJKl3796yWCzW56dOndKjjz6qatWqyd3dXc2aNdOmTZtsjpeZmannn39evr6+cnFxUUBAgKKjo/Os7fr16woLC1Pr1q0LXDLj4uIib29vm4eDg4PmzJmj4OBglStXTr6+vho9erTS0tJszu+3y2Py8v7776tu3bpydXVVUFCQ3nnnHZv2PXv2qHHjxnJ1dVXTpk319ddfFzje3UJoBwAAKEVWrFihoKAg1alTR4MHD9bChQtlGIakm8tHnn32WdWvX986g92/f3/t3btXkrRo0SKdP3/e+jwtLU3du3fX5s2b9fXXX6tbt27q2bOnzpw5Yz3ekCFD9OGHH2revHk6cuSI3nvvPbm7u+eqKzk5WV26dFFOTo42btx423CdlzJlymjevHn65ptvtHjxYm3ZskV/+9vfCr1/bGyspk6dqldeeUVHjhzRq6++qilTpliX46SlpalHjx6qV6+e9u/fr+nTp2vSpElFrrMksKYdAACgFImOjtbgwYMl3VxmkpKSooSEBLVv315ubm5yd3eXo6OjvL29rfu4ublJkry8vGy2N2rUSI0aNbI+f+mllxQXF6dPPvlEY8eO1fHjx7VixQpt3LhRnTt3liTVqlUrV00XLlxQ//79FRgYqKVLl8rZ2bnAc1i7dq1N8A8LC9PKlSs1fvx46zY/Pz+9/PLLGjVqVK7Z8vxMmzZNs2fPVp8+fSRJ/v7++vbbb/Xee+8pMjJSS5cuVU5OjqKjo+Xq6qr69evrhx9+0FNPPVWo8UsSoR0AAKCUOHbsmPbs2aO4uDhJkqOjo/r376/o6Gi1b9++yOOlpaVp+vTpWrdunc6fP68bN27ol19+sc60JyYmysHBQe3atStwnC5duqh58+Zavny5HBwcbnvcDh06aMGCBdbn5cqVkyRt2rRJM2bM0NGjR5WamqobN24oIyND169fV9myZQscMz09XadOndLw4cM1cuRI6/YbN27I0/PmB4yPHDmihg0bytXV1dreqlWr29Z7NxDaAQAASono6GjduHFDPj4+1m2GYcjFxUVvv/22NZwW1qRJk7Rx40bNmjVLAQEBcnNzU9++fZWVlSXp/2bobyc8PFwfffSRvv32WwUHB9+2f7ly5XLdKeb06dPq0aOHnnrqKb3yyiuqWLGivvjiCw0fPlxZWVm3De231r7/5z//UYsWLWzaCvOLhL0R2gEAAEqBGzduaMmSJZo9e7a6du1q09arVy99+OGHGjVqlJydnZWdnZ1rfycnp1zbd+zYoaFDh6p3796Sbgbf06dPW9uDg4OVk5OjhIQE6/KYvMycOVPu7u7q1KmT4uPjVa9evSKf3/79+5WTk6PZs2erTJmbH8tcsWJFofevVq2afHx89N1332nQoEF59qlbt64++OADZWRkWGfbd+3aVeRaSwIfRAUAACgF1q5dq6tXr2r48OFq0KCBzSMiIsJ6Rxc/Pz8lJSUpMTFRly9fVmZmpnX75s2bdeHCBV29elWSFBgYqNWrVysxMVEHDhzQ448/rpycHOsx/fz8FBkZqSeffFJr1qxRUlKS4uPj8wzTs2bN0qBBg9SxY0cdPXq0yOcXEBCgX3/9VW+99Za+++47ffDBB3r33XeLNEZUVJRmzJihefPm6fjx4zp06JAWLVqkOXPmSJIef/xxWSwWjRw5Ut9++63Wr1+vWbNmFbnWkkBoBwAAKAWio6PVuXPnPJfAREREaN++fTp48KAiIiLUrVs3dejQQVWqVNGHH34oSZo9e7Y2btwoX19fNW7cWJI0Z84cVahQQQ899JB69uyp0NBQNWnSxGbsBQsWqG/fvho9erSCgoI0cuRIpaen51nj3Llz1a9fP3Xs2FHHjx8v0vk1atRIc+bM0WuvvaYGDRooNjZWM2bMKNIYI0aM0Pvvv69FixYpODhY7dq1U0xMjPUe8O7u7vr000916NAhNW7cWC+88IJee+21Ih2jpFiMW/cAKqVSU1Pl6emplJQUeXh42KeIwnx72h1+4xkAAKWRvf/9zsjIUFJSkvz9/W0+lAgUt8K+15hpBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAIASER8fL4vFouTk5GId12KxaM2aNfm2+/n56c033yzWY96p9u3ba/z48X94HMc/XgoAAMCfQ/Di4Lt6vEORh+5ov507d+rhhx9Wt27dtG7dOpu26dOna82aNUpMTLTZbrFYFBcXp169et1htcXjp59+0tSpU7Vu3TpdvHhRFSpUUKNGjTR16lS1bt26UGPs3btX5cqVK+FK7y5COwAAQCkTHR2tcePGKTo6WufOnZOPj4+9Syq0iIgIZWVlafHixapVq5YuXryozZs368qVK4Ueo0qVKiVYoX2wPAYAAKAUSUtL0/Lly/XUU08pPDxcMTEx1raYmBhFRUXpwIEDslgsslgsiomJkZ+fnySpd+/eslgs1uenTp3So48+qmrVqsnd3V3NmjXTpk2bbI6XmZmp559/Xr6+vnJxcVFAQICio6PzrO369esKCwtT69at81wyk5ycrO3bt+u1115Thw4dVLNmTTVv3lyTJ0/WI488ku85T5s2TdWrV9fBgwcl5V4eY7FYtGDBAoWFhcnNzU21atXSqlWrrO2nT5+WxWLRsmXL9NBDD8nV1VUNGjRQQkKCzXEOHz6ssLAwubu7q1q1anriiSd0+fJla3t6erqGDBkid3d3Va9eXbNnz8635qIitAMAAJQiK1asUFBQkOrUqaPBgwdr4cKFMgxDktS/f389++yzql+/vs6fP6/z58+rf//+2rt3ryRp0aJFOn/+vPV5Wlqaunfvrs2bN+vrr79Wt27d1LNnT505c8Z6vCFDhujDDz/UvHnzdOTIEb333ntyd3fPVVdycrK6dOminJwcbdy4UV5eXrn6uLu7y93dXWvWrFFmZuZtz9UwDI0bN05LlizR9u3b1bBhw3z7TpkyRRERETpw4IAGDRqkAQMG6MiRIzZ9nnvuOT377LP6+uuv1apVK/Xs2dM6w5+cnKyOHTuqcePG2rdvnzZs2KCLFy+qX79+NvsnJCTo448/1v/+9z/Fx8frq6++uu15FAahHQAAoBSJjo7W4MGDJUndunVTSkqKdcbYzc1N7u7ucnR0lLe3t7y9veXm5mZdTuLl5SVvb2/r80aNGumvf/2rGjRooMDAQL300kt64IEH9Mknn0iSjh8/rhUrVmjhwoXq3bu3atWqpU6dOql///42NV24cEHt2rVT9erV9emnn6ps2bJ51u7o6KiYmBgtXrxYXl5eat26tf7xj39YZ9B/68aNGxo8eLA2b96sL774QgEBAQVel8cee0wjRoxQ7dq19dJLL6lp06Z66623bPqMHTtWERERqlu3rhYsWCBPT0/rXw3efvttNW7cWK+++qqCgoLUuHFjLVy4UFu3btXx48eVlpam6OhozZo1S506dVJwcLAWL16sGzduFFhXYRHaAQAASoljx45pz549GjhwoKSbIbh///75Lle5nbS0NE2aNEl169aVl5eX3N3ddeTIEetMe2JiohwcHNSuXbsCx+nSpYsCAgK0fPlyOTs7F9g3IiJC586d0yeffKJu3bopPj5eTZo0sVnmI0kTJkzQ7t27tW3bNt133323PZdWrVrlev77mfbf9nF0dFTTpk2tfQ4cOKCtW7da/xrg7u6uoKAgSTeXEZ06dUpZWVlq0aKFdYyKFSuqTp06t62tMAjtAAAApUR0dLRu3LghHx8fOTo6ytHRUQsWLNBHH32klJSUIo83adIkxcXF6dVXX9X27duVmJio4OBgZWVlSbo5c18Y4eHh2rZtm7799ttC9Xd1dVWXLl00ZcoUffnllxo6dKimTZtm06dLly768ccf9fnnnxftpO5QWlqaevbsqcTERJvHiRMn1LZt2xI/PqEdAACgFLhx44aWLFmi2bNn24TKAwcOyMfHRx9++KEkydnZWdnZ2bn2d3JyyrV9x44dGjp0qHr37q3g4GB5e3vr9OnT1vbg4GDl5OTk+sDm782cOVORkZHq1KlToYP7b9WrV0/p6ek22x555BEtXbpUI0aM0LJly247xq5du3I9r1u3br59bty4of3791v7NGnSRN988438/PwUEBBg8yhXrpweeOABOTk5affu3dYxrl69quPHjxf5fPNCaAcAACgF1q5dq6tXr2r48OFq0KCBzSMiIsK6RMbPz09JSUlKTEzU5cuXrR/49PPz0+bNm3XhwgVdvXpVkhQYGKjVq1dbw//jjz+unJwc6zH9/PwUGRmpJ598UmvWrFFSUpLi4+O1YsWKXPXNmjVLgwYNUseOHXX06NE8z+HKlSvq2LGj/vvf/+rgwYNKSkrSypUr9frrr+vRRx/N1b9379764IMPNGzYMJu7weRl5cqVWrhwoY4fP65p06Zpz549Gjt2rE2f+fPnKy4uTkePHtWYMWN09epVPfnkk5KkMWPG6Oeff9bAgQO1d+9enTp1Sp9//rmGDRum7Oxsubu7a/jw4Xruuee0ZcsWHT58WEOHDlWZMsUTtwntAAAApUB0dLQ6d+4sT0/PXG0RERHat2+fDh48qIiICHXr1k0dOnRQlSpVrDPws2fP1saNG+Xr66vGjRtLkubMmaMKFSrooYceUs+ePRUaGqomTZrYjL1gwQL17dtXo0ePVlBQkEaOHJlrVvyWuXPnql+/furYsWOeM9Du7u5q0aKF5s6dq7Zt26pBgwaaMmWKRo4cqbfffjvPMfv27avFixfriSee0OrVq/O9PlFRUVq2bJkaNmyoJUuW6MMPP1S9evVs+sycOVMzZ85Uo0aN9MUXX+iTTz5R5cqVJUk+Pj7asWOHsrOz1bVrVwUHB2v8+PHy8vKyBvM33nhDbdq0Uc+ePdW5c2c9/PDDCgkJybemorAYt+4BVEqlpqbK09NTKSkp8vDwsE8R03P/8OTuU/R1ZgAAlFb2/vc7IyNDSUlJ8vf3l6ur610/PorX7b7t9fTp0/L399fXX3+tBx988K7WVtj3GjPtAAAAgMkR2gEAAACTc7R3AQAAAEBJut1qcD8/v9v2sTdm2gEAAACTI7QDAADkw+yzr7j3FfY9ZtfQPmPGDDVr1kzly5dX1apV1atXLx07dsymT/v27WWxWGweo0aNslPFAADgz8DBwUGSrN/8CZSU69evS7r55VYFseua9oSEBI0ZM0bNmjXTjRs39I9//ENdu3bVt99+q3Llyln7jRw5Ui+++KL1edmyZe1RLgAA+JNwdHRU2bJl9dNPP8nJyanYviAHuMUwDF2/fl2XLl2Sl5eX9RfF/Ng1tG/YsMHmeUxMjKpWrar9+/erbdu21u1ly5aVt7f33S4PAAD8SVksFlWvXl1JSUn6/vvv7V0OSjEvL69C5VxT3T0mJeXmFwxVrFjRZntsbKz++9//ytvbWz179tSUKVPynW3PzMy0fh2vdPPLGQAAAIrK2dlZgYGBLJFBiXFycrrtDPstpgntOTk5Gj9+vFq3bq0GDRpYtz/++OOqWbOmfHx8dPDgQT3//PM6duxYvl9TO2PGDEVFRd2tsgEAQClWpkwZvhEVpmAxTPKx6KeeekqfffaZvvjiC9WoUSPfflu2bFGnTp108uRJPfDAA7na85pp9/X1tdvXIEuSpnsWok9KydcBAMA9IjU1VZ6envb99xswEVPMtI8dO1Zr167Vtm3bCgzsktSiRQtJyje0u7i4yMXFpUTqBAAAAOzBrqHdMAyNGzdOcXFxio+Pl7+//233SUxMlCRVr169hKsDAAAAzMGuoX3MmDFaunSpPv74Y5UvX14XLlyQJHl6esrNzU2nTp3S0qVL1b17d1WqVEkHDx7UhAkT1LZtWzVs2NCepQMAAAB3jV1D+4IFCyTd/AKl31q0aJGGDh0qZ2dnbdq0SW+++abS09Pl6+uriIgI/fOf/7RDtQAAAIB92H15TEF8fX2VkJBwl6oBAAAAzImv9wIAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACZHaAcAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACZHaAcAAABMztHeBQB/Jn5/X1eofqdnhpdwJQAA4F7CTDsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACZHaAcAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJOdq7AABA6eT393WF6nd6ZngJV1J093LtAEonZtoBAAAAkyO0AwAAACZHaAcAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACbnaO8CAKCk3ctfSX8v1w4AKD7MtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAydk1tM+YMUPNmjVT+fLlVbVqVfXq1UvHjh2z6ZORkaExY8aoUqVKcnd3V0REhC5evGinigEAAIC7z66hPSEhQWPGjNGuXbu0ceNG/frrr+ratavS09OtfSZMmKBPP/1UK1euVEJCgs6dO6c+ffrYsWoAAADg7rLrN6Ju2LDB5nlMTIyqVq2q/fv3q23btkpJSVF0dLSWLl2qjh07SpIWLVqkunXrateuXWrZsqU9ygYAAADuKlOtaU9JSZEkVaxYUZK0f/9+/frrr+rcubO1T1BQkO6//37t3LkzzzEyMzOVmppq8wAAAADuZXadaf+tnJwcjR8/Xq1bt1aDBg0kSRcuXJCzs7O8vLxs+larVk0XLlzIc5wZM2YoKiqqpMu95/n9fV2h+p2eGV7ClRTdvVz7vYzrDgCA/Zhmpn3MmDE6fPiwli1b9ofGmTx5slJSUqyPs2fPFlOFAAAAgH2YYqZ97NixWrt2rbZt26YaNWpYt3t7eysrK0vJyck2s+0XL16Ut7d3nmO5uLjIxcWlpEsGAAAA7hq7zrQbhqGxY8cqLi5OW7Zskb+/v017SEiInJyctHnzZuu2Y8eO6cyZM2rVqtXdLhcAAACwC7vOtI8ZM0ZLly7Vxx9/rPLly1vXqXt6esrNzU2enp4aPny4Jk6cqIoVK8rDw0Pjxo1Tq1atuHMMAAAA/jTsGtoXLFggSWrfvr3N9kWLFmno0KGSpLlz56pMmTKKiIhQZmamQkND9c4779zlSgEAAAD7sWtoNwzjtn1cXV01f/58zZ8//y5UBAAAAJiPae4eAwAAACBvhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJmeKb0QFAADFw+/v6wrV7/TM8BKuBEBxYqYdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5O4otNeqVUtXrlzJtT05OVm1atX6w0UBAAAA+D93FNpPnz6t7OzsXNszMzP1448//uGiAAAAAPwfx6J0/uSTT6z//fnnn8vT09P6PDs7W5s3b5afn1+xFQcAAACgiKG9V69ekiSLxaLIyEibNicnJ/n5+Wn27NnFVhwAAACAIob2nJwcSZK/v7/27t2rypUrl0hRAAAAAP5PkUL7LUlJScVdBwAAAIB83FFol6TNmzdr8+bNunTpknUG/paFCxf+4cIAAAAA3HRHoT0qKkovvviimjZtqurVq8tisRR3XQAAAAD+vzsK7e+++65iYmL0xBNPFHc9AAAAAH7nju7TnpWVpYceeqi4awEAAACQhzsK7SNGjNDSpUuLuxYAAAAAebij5TEZGRn697//rU2bNqlhw4ZycnKyaZ8zZ06xFAcAAADgDkP7wYMH9eCDD0qSDh8+bNPGh1IBAACA4nVHoX3r1q3FXQcAAACAfNzRmnYAAAAAd88dzbR36NChwGUwW7ZsueOCAAAAANi6o9B+az37Lb/++qsSExN1+PBhRUZGFkddAAAAAP6/Owrtc+fOzXP79OnTlZaW9ocKAgAAAGCrWNe0Dx48WAsXLizOIQEAAIA/vWIN7Tt37pSrq2txDgkAAAD86d3R8pg+ffrYPDcMQ+fPn9e+ffs0ZcqUYikMAAAAwE13FNo9PT1tnpcpU0Z16tTRiy++qK5duxZLYQAAAABuuqPQvmjRomI5+LZt2/TGG29o//79On/+vOLi4tSrVy9r+9ChQ7V48WKbfUJDQ7Vhw4ZiOT4AAABwL7ij0H7L/v37deTIEUlS/fr11bhx4yLtn56erkaNGunJJ5/MteTmlm7dutn8kuDi4nLnBQMAAAD3oDsK7ZcuXdKAAQMUHx8vLy8vSVJycrI6dOigZcuWqUqVKoUaJywsTGFhYQX2cXFxkbe3952UCQAAAJQKd3T3mHHjxunatWv65ptv9PPPP+vnn3/W4cOHlZqaqqeffrpYC4yPj1fVqlVVp04dPfXUU7py5Uqxjg8AAACY3R3NtG/YsEGbNm1S3bp1rdvq1aun+fPnF+sHUbt166Y+ffrI399fp06d0j/+8Q+FhYVp586dcnBwyHOfzMxMZWZmWp+npqYWWz0AAACAPdxRaM/JyZGTk1Ou7U5OTsrJyfnDRd0yYMAA638HBwerYcOGeuCBBxQfH69OnTrluc+MGTMUFRVVbDUAAAAA9nZHy2M6duyoZ555RufOnbNu+/HHHzVhwoR8w3RxqFWrlipXrqyTJ0/m22fy5MlKSUmxPs6ePVti9QAAAAB3wx3NtL/99tt65JFH5OfnJ19fX0nS2bNn1aBBA/33v/8t1gJ/64cfftCVK1dUvXr1fPu4uLhwhxkAAACUKncU2n19ffXVV19p06ZNOnr0qCSpbt266ty5c5HGSUtLs5k1T0pKUmJioipWrKiKFSsqKipKERER8vb21qlTp/S3v/1NAQEBCg0NvZOyAQAAgHtSkZbHbNmyRfXq1VNqaqosFou6dOmicePGady4cWrWrJnq16+v7du3F3q8ffv2qXHjxtb7u0+cOFGNGzfW1KlT5eDgoIMHD+qRRx5R7dq1NXz4cIWEhGj79u3MpAMAAOBPpUgz7W+++aZGjhwpDw+PXG2enp7661//qjlz5qhNmzaFGq99+/YyDCPf9s8//7wo5QEAAAClUpFm2g8cOKBu3brl2961a1ft37//DxcFAAAA4P8UKbRfvHgxz1s93uLo6KiffvrpDxcFAAAA4P8UKbTfd999Onz4cL7tBw8eLPDOLgAAAACKrkihvXv37poyZYoyMjJytf3yyy+aNm2aevToUWzFAQAAACjiB1H/+c9/avXq1apdu7bGjh2rOnXqSJKOHj2q+fPnKzs7Wy+88EKJFAoAAAD8WRUptFerVk1ffvmlnnrqKU2ePNl65xeLxaLQ0FDNnz9f1apVK5FCAQAAgD+rIn+5Us2aNbV+/XpdvXpVJ0+elGEYCgwMVIUKFUqiPgAAAOBP746+EVWSKlSooGbNmhVnLQAAAADyUKQPogIAAAC4+wjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGBydg3t27ZtU8+ePeXj4yOLxaI1a9bYtBuGoalTp6p69epyc3NT586ddeLECfsUCwAAANiJXUN7enq6GjVqpPnz5+fZ/vrrr2vevHl69913tXv3bpUrV06hoaHKyMi4y5UCAAAA9uNoz4OHhYUpLCwszzbDMPTmm2/qn//8px599FFJ0pIlS1StWjWtWbNGAwYMuJulAgAAAHZj2jXtSUlJunDhgjp37mzd5unpqRYtWmjnzp357peZmanU1FSbBwAAAHAvM21ov3DhgiSpWrVqNturVatmbcvLjBkz5OnpaX34+vqWaJ0AAABASTNtaL9TkydPVkpKivVx9uxZe5cEAAAA/CGmDe3e3t6SpIsXL9psv3jxorUtLy4uLvLw8LB5AAAAAPcy04Z2f39/eXt7a/PmzdZtqamp2r17t1q1amXHygAAAIC7y653j0lLS9PJkyetz5OSkpSYmKiKFSvq/vvv1/jx4/Xyyy8rMDBQ/v7+mjJlinx8fNSrVy/7FQ0AAADcZXYN7fv27VOHDh2szydOnChJioyMVExMjP72t78pPT1df/nLX5ScnKyHH35YGzZskKurq71KBgAAAO46u4b29u3byzCMfNstFotefPFFvfjii3exKgAAAMBcTLumHQAAAMBNhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJudo7wIAlDLTPQvRJ6Xk67gT93LtQFHxfgfuKcy0AwAAACZHaAcAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACZHaAcAAABMztHeBQAlhq/oBlDS+P8MgLuEmXYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlTh/bp06fLYrHYPIKCguxdFgAAAHBXmf7LlerXr69NmzZZnzs6mr5kAAAAoFiZPgE7OjrK29vb3mUAAAAAdmPq5TGSdOLECfn4+KhWrVoaNGiQzpw5U2D/zMxMpaam2jwAAACAe5mpZ9pbtGihmJgY1alTR+fPn1dUVJTatGmjw4cPq3z58nnuM2PGDEVFRd3lSkux6Z637RLsf3+hhjoUeeiPVgMgP4X4WdX0lJKv407cy7UDwF1i6pn2sLAwPfbYY2rYsKFCQ0O1fv16JScna8WKFfnuM3nyZKWkpFgfZ8+evYsVAwAAAMXP1DPtv+fl5aXatWvr5MmT+fZxcXGRi4vLXawKAAAAKFmmnmn/vbS0NJ06dUrVq1e3dykAAADAXWPq0D5p0iQlJCTo9OnT+vLLL9W7d285ODho4MCB9i4NAAAAuGtMvTzmhx9+0MCBA3XlyhVVqVJFDz/8sHbt2qUqVarYuzQAAADgrjF1aF+2bJm9SwAAAADsztTLYwAAAAAQ2gEAAADTI7QDAAAAJkdoBwAAAEyO0A4AAACYnKnvHgOUtODFwYXqdyjyUAlX8jvTPQvRJ6Xk60CpUpj3+11/r8PUeM8A5sFMOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMk52rsAAHcmeHFwofodijxUwpUU3b1cO+yjMO8Z3i8ASjNm2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJudo7wIA4F4SvDj4tn0ORR66C5XgXsF7BkBxYKYdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGBy90Ronz9/vvz8/OTq6qoWLVpoz5499i4JAAAAuGtMH9qXL1+uiRMnatq0afrqq6/UqFEjhYaG6tKlS/YuDQAAALgrTB/a58yZo5EjR2rYsGGqV6+e3n33XZUtW1YLFy60d2kAAADAXWHq0J6VlaX9+/erc+fO1m1lypRR586dtXPnTjtWBgAAANw9jvYuoCCXL19Wdna2qlWrZrO9WrVqOnr0aJ77ZGZmKjMz0/o8JSVFkpSamlpyhd5OpnH7Pne5vpzM64Xql2q5fe3Zv2QXbqxiOkdqv4naC+9u116c/7+h9puovfDu5drzGtcwCvFvKPAnYDFM/NNw7tw53Xffffryyy/VqlUr6/a//e1vSkhI0O7du3PtM336dEVFRd3NMgEAQAk5e/asatSoYe8yALsz9Ux75cqV5eDgoIsXL9psv3jxory9vfPcZ/LkyZo4caL1eU5Ojn7++WdVqlRJFoul2GpLTU2Vr6+vzp49Kw8Pj2IbFwXjutsH190+uO72wXW3j99fd8MwdO3aNfn4+Ni7NMAUTB3anZ2dFRISos2bN6tXr16SbobwzZs3a+zYsXnu4+LiIhcXF5ttXl5eJVajh4cH/1O3A667fXDd7YPrbh9cd/v47XX39PS0czWAeZg6tEvSxIkTFRkZqaZNm6p58+Z68803lZ6ermHDhtm7NAAAAOCuMH1o79+/v3766SdNnTpVFy5c0IMPPqgNGzbk+nAqAAAAUFqZPrRL0tixY/NdDmMvLi4umjZtWq6lOChZXHf74LrbB9fdPrju9sF1Bwpm6rvHAAAAADD5lysBAAAAILQDAAAApkdoBwAAAEyO0A4AAACYHKH9Ds2fP19+fn5ydXVVixYttGfPHnuXVKps27ZNPXv2lI+PjywWi9asWWPTbhiGpk6dqurVq8vNzU2dO3fWiRMn7FNsKTJjxgw1a9ZM5cuXV9WqVdWrVy8dO3bMpk9GRobGjBmjSpUqyd3dXREREbm+tRhFs2DBAjVs2ND6pTKtWrXSZ599Zm3nmpe8mTNnymKxaPz48dZtXPfiN336dFksFptHUFCQtZ1rDuSP0H4Hli9frokTJ2ratGn66quv1KhRI4WGhurSpUv2Lq3USE9PV6NGjTR//vw8219//XXNmzdP7777rnbv3q1y5copNDRUGRkZd7nS0iUhIUFjxozRrl27tHHjRv3666/q2rWr0tPTrX0mTJigTz/9VCtXrlRCQoLOnTunPn362LHqe1+NGjU0c+ZM7d+/X/v27VPHjh316KOP6ptvvpHENS9pe/fu1XvvvaeGDRvabOe6l4z69evr/Pnz1scXX3xhbeOaAwUwUGTNmzc3xowZY32enZ1t+Pj4GDNmzLBjVaWXJCMuLs76PCcnx/D29jbeeOMN67bk5GTDxcXF+PDDD+1QYel16dIlQ5KRkJBgGMbN6+zk5GSsXLnS2ufIkSOGJGPnzp32KrNUqlChgvH+++9zzUvYtWvXjMDAQGPjxo1Gu3btjGeeecYwDN7rJWXatGlGo0aN8mzjmgMFY6a9iLKysrR//3517tzZuq1MmTLq3Lmzdu7cacfK/jySkpJ04cIFm9fA09NTLVq04DUoZikpKZKkihUrSpL279+vX3/91ebaBwUF6f777+faF5Ps7GwtW7ZM6enpatWqFde8hI0ZM0bh4eE211fivV6STpw4IR8fH9WqVUuDBg3SmTNnJHHNgdu5J74R1UwuX76s7OxsVatWzWZ7tWrVdPToUTtV9edy4cIFScrzNbjVhj8uJydH48ePV+vWrdWgQQNJN6+9s7OzvLy8bPpy7f+4Q4cOqVWrVsrIyJC7u7vi4uJUr149JSYmcs1LyLJly/TVV19p7969udp4r5eMFi1aKCYmRnXq1NH58+cVFRWlNm3a6PDhw1xz4DYI7QDyNGbMGB0+fNhmvSlKTp06dZSYmKiUlBStWrVKkZGRSkhIsHdZpdbZs2f1zDPPaOPGjXJ1dbV3OX8aYWFh1v9u2LChWrRooZo1a2rFihVyc3OzY2WA+bE8pogqV64sBweHXJ9mv3jxory9ve1U1Z/LrevMa1Byxo4dq7Vr12rr1q2qUaOGdbu3t7eysrKUnJxs059r/8c5OzsrICBAISEhmjFjhho1aqR//etfXPMSsn//fl26dElNmjSRo6OjHB0dlZCQoHnz5snR0VHVqlXjut8FXl5eql27tk6ePMl7HbgNQnsROTs7KyQkRJs3b7Zuy8nJ0ebNm9WqVSs7Vvbn4e/vL29vb5vXIDU1Vbt37+Y1+IMMw9DYsWMVFxenLVu2yN/f36Y9JCRETk5ONtf+2LFjOnPmDNe+mOXk5CgzM5NrXkI6deqkQ4cOKTEx0fpo2rSpBg0aZP1vrnvJS0tL06lTp1S9enXe68BtsDzmDkycOFGRkZFq2rSpmjdvrjfffFPp6ekaNmyYvUsrNdLS0nTy5Enr86SkJCUmJqpixYq6//77NX78eL388ssKDAyUv7+/pkyZIh8fH/Xq1ct+RZcCY8aM0dKlS/Xxxx+rfPny1nWknp6ecnNzk6enp4YPH66JEyeqYsWK8vDw0Lhx49SqVSu1bNnSztXfuyZPnqywsDDdf//9unbtmpYuXar4+Hh9/vnnXPMSUr58eetnNW4pV66cKlWqZN3OdS9+kyZNUs+ePVWzZk2dO3dO06ZNk4ODgwYOHMh7Hbgde9++5l711ltvGffff7/h7OxsNG/e3Ni1a5e9SypVtm7dakjK9YiMjDQM4+ZtH6dMmWJUq1bNcHFxMTp16mQcO3bMvkWXAnldc0nGokWLrH1++eUXY/To0UaFChWMsmXLGr179zbOnz9vv6JLgSeffNKoWbOm4ezsbFSpUsXo1KmT8b///c/azjW/O357y0fD4LqXhP79+xvVq1c3nJ2djfvuu8/o37+/cfLkSWs71xzIn8UwDMNOvy8AAAAAKATWtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAP4U4qPj5fFYlFycnKR9rty5YqqVq2q06dPF2qcy5cvq2rVqvrhhx/+WMEAgD81QjsAG0OHDlWvXr3sdnw/Pz9ZLBZZLBaVLVtWwcHBev/99+1Wz++98sorevTRR+Xn51eo/pUrV9aQIUM0bdq0ki0MAFCqEdoBmM6LL76o8+fP6/Dhwxo8eLBGjhypzz77zN5l6fr164qOjtbw4cOLtN+wYcMUGxurn3/+uYQqAwCUdoR2AEWSkJCg5s2by8XFRdWrV9ff//533bhxQ5K0du1aeXl5KTs7W5KUmJgoi8Wiv//979b9R4wYocGDBxd4jPLly8vb21u1atXS888/r4oVK2rjxo3W9r1796pLly6qXLmyPD091a5dO3311Vc2Y1gsFr3//vvq3bu3ypYtq8DAQH3yySf5HvP69esKCwtT69at813qsn79erm4uKhly5ZFGqd+/fry8fFRXFxcgecNAEB+CO0ACu3HH39U9+7d1axZMx04cEALFixQdHS0Xn75ZUlSmzZtdO3aNX399deSbgb8ypUrKz4+3jpGQkKC2rdvX6jj5eTk6KOPPtLVq1fl7Oxs3X7t2jVFRkbqiy++0K5duxQYGKju3bvr2rVrNvtHRUWpX79+OnjwoLp3765BgwblOdudnJysLl26KCcnRxs3bpSXl1ee9Wzfvl0hISH51lvQOM2bN9f27dsLdd4AAPweoR1Aob3zzjvy9fXV22+/raCgIPXq1UtRUVGaPXu2cnJy5OnpqQcffNAa0uPj4zVhwgR9/fXXSktL048//qiTJ0+qXbt2BR7n+eefl7u7u1xcXNS3b19VqFBBI0aMsLZ37NhRgwcPVlBQkOrWrat///vfun79uhISEmzGGTp0qAYOHKiAgAC9+uqrSktL0549e2z6XLhwQe3atVP16tX16aefqmzZsvnW9f3338vHxyfPttuN4+Pjo++//77A8wYAID+EdgCFduTIEbVq1UoWi8W6rXXr1kpLS7PeHaVdu3aKj4+XYRjavn27+vTpo7p16+qLL75QQkKCfHx8FBgYWOBxnnvuOSUmJmrLli1q0aKF5s6dq4CAAGv7xYsXNXLkSAUGBsrT01MeHh5KS0vTmTNnbMZp2LCh9b/LlSsnDw8PXbp0yaZPly5dFBAQoOXLl9vM5ufll19+kaura55ttxvHzc1N169fL3B8AADyQ2gHUKzat2+vL774QgcOHJCTk5OCgoLUvn17xcfHKyEh4baz7NLNO64EBASoTZs2WrlypZ5++ml9++231vbIyEglJibqX//6l7788kslJiaqUqVKysrKshnHycnJ5rnFYlFOTo7NtvDwcG3bts1m/ILqunr1ap5ttxvn559/VpUqVW57DAAA8kJoB1BodevW1c6dO2UYhnXbjh07VL58edWoUUPS/61rnzt3rjWg3wrt8fHxhV7Pfouvr6/69++vyZMn2xzz6aefVvfu3VW/fn25uLjo8uXLd3ROM2fOVGRkpDp16nTb4N64ceN8+9xunMOHD6tx48Z3VCMAAIR2ALmkpKQoMTHR5nH27FmNHj1aZ8+e1bhx43T06FF9/PHHmjZtmiZOnKgyZW7+76RChQpq2LChYmNjrQG9bdu2+uqrr3T8+PFCzbT/3jPPPKNPP/1U+/btkyQFBgbqgw8+0JEjR7R7924NGjRIbm5ud3y+s2bN0qBBg9SxY0cdPXo0336hoaH65ptv8p1tz2+c69eva//+/eratesd1wgA+HMjtAPIJT4+Xo0bN7Z5REVF6b777tP69eu1Z88eNWrUSKNGjdLw4cP1z3/+02b/du3aKTs72xraK1asqHr16snb21t16tQpcj316tVT165dNXXqVElSdHS0rl69qiZNmuiJJ57Q008/rapVq/6hc547d6769eunjh076vjx43n2CQ4OVpMmTbRixYoijfPxxx/r/vvvV5s2bf5QjQCAPy+L8du/cwMACrRu3To999xzOnz4sPWvC7fTsmVLPf3003r88cdLuDoAQGnlaO8CAOBeEh4erhMnTujHH3+Ur6/vbftfvnxZffr00cCBA+9CdQCA0oqZdgAAAMDkWNMOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACZHaAcAAABMjtAOAAAAmNz/A9HYfHSgXAYiAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bar_width = 1\n",
    "plt.bar(np.arange(1, 50, 5) - bar_width/2, success, width = bar_width, label=\"Attack Success\")\n",
    "plt.bar(np.arange(1, 50, 5) + bar_width/2, failed, width = bar_width, label=\"Attack Failed\")\n",
    "plt.bar(np.arange(1, 50, 5) + bar_width + bar_width/2, skipped, width = bar_width, label = \"Attack Skipped\")\n",
    "plt.title(\"Attack on Self Attention Matrix\")\n",
    "plt.xlabel(\"Low Rank (k)\")\n",
    "plt.ylabel(\"Count\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1.0), loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd07f097-8b01-4314-b803-2801e0e543ac",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b55d48f-e993-4063-9179-42adf2be051b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (Local)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}