src/Train-Inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "eb31357e",
   "metadata": {},
   "source": [
    "# SemEval-2024"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "990b8d38",
   "metadata": {},
   "source": [
    "# Scorer from organizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "53bf4faa",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pdb\n",
    "import json\n",
    "import logging.handlers\n",
    "import argparse\n",
    "import os\n",
    "import numpy as np\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "from networkx import DiGraph, relabel_nodes, all_pairs_shortest_path_length\n",
    "from sklearn_hierarchical_classification.constants import ROOT\n",
    "from sklearn_hierarchical_classification.metrics import h_fbeta_score, h_recall_score, h_precision_score, \\\n",
    "    fill_ancestors, multi_labeled\n",
    "import sys\n",
    "sys.path.append('.')\n",
    "\n",
    "\n",
    "KEYS = ['id','labels']\n",
    "logger = logging.getLogger(\"subtask_1_2a_scorer\")\n",
    "ch = logging.StreamHandler(sys.stdout)\n",
    "ch.setLevel(logging.INFO)\n",
    "formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')\n",
    "ch.setFormatter(formatter)\n",
    "logger.setLevel(logging.INFO)\n",
    "#logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)\n",
    "\n",
    "G = DiGraph()\n",
    "G.add_edge(ROOT, \"Logos\")\n",
    "G.add_edge(\"Logos\", \"Repetition\")\n",
    "G.add_edge(\"Logos\", \"Obfuscation, Intentional vagueness, Confusion\")\n",
    "G.add_edge(\"Logos\", \"Reasoning\")\n",
    "G.add_edge(\"Logos\", \"Justification\")\n",
    "G.add_edge('Justification', \"Slogans\")\n",
    "G.add_edge('Justification', \"Bandwagon\")\n",
    "G.add_edge('Justification', \"Appeal to authority\")\n",
    "G.add_edge('Justification', \"Flag-waving\")\n",
    "G.add_edge('Justification', \"Appeal to fear/prejudice\")\n",
    "G.add_edge('Reasoning', \"Simplification\")\n",
    "G.add_edge('Simplification', \"Causal Oversimplification\")\n",
    "G.add_edge('Simplification', \"Black-and-white Fallacy/Dictatorship\")\n",
    "G.add_edge('Simplification', \"Thought-terminating cliché\")\n",
    "G.add_edge('Reasoning', \"Distraction\")\n",
    "G.add_edge('Distraction', \"Misrepresentation of Someone's Position (Straw Man)\")\n",
    "G.add_edge('Distraction', \"Presenting Irrelevant Data (Red Herring)\")\n",
    "G.add_edge('Distraction', \"Whataboutism\")\n",
    "G.add_edge(ROOT, \"Ethos\")\n",
    "G.add_edge('Ethos', \"Appeal to authority\")\n",
    "G.add_edge('Ethos', \"Glittering generalities (Virtue)\")\n",
    "G.add_edge('Ethos', \"Bandwagon\")\n",
    "G.add_edge('Ethos', \"Ad Hominem\")\n",
    "G.add_edge('Ethos', \"Transfer\")\n",
    "G.add_edge('Ad Hominem', \"Doubt\")\n",
    "G.add_edge('Ad Hominem', \"Name calling/Labeling\")\n",
    "G.add_edge('Ad Hominem', \"Smears\")\n",
    "G.add_edge('Ad Hominem', \"Reductio ad hitlerum\")\n",
    "G.add_edge('Ad Hominem', \"Whataboutism\")\n",
    "G.add_edge(ROOT, \"Pathos\")\n",
    "G.add_edge('Pathos', \"Exaggeration/Minimisation\")\n",
    "G.add_edge('Pathos', \"Loaded Language\")\n",
    "G.add_edge('Pathos', \"Appeal to (Strong) Emotions\")\n",
    "G.add_edge('Pathos', \"Appeal to fear/prejudice\")\n",
    "G.add_edge('Pathos', \"Flag-waving\")\n",
    "G.add_edge('Pathos', \"Transfer\") \n",
    "\n",
    "def get_all_classes_from_graph(graph):\n",
    "    return [\n",
    "        node\n",
    "        for node in graph.nodes\n",
    "        if node != ROOT\n",
    "        ]\n",
    "    \n",
    "def _h_fbeta_score(y_true, y_pred, class_hierarchy, beta=1., root=ROOT):\n",
    "    hP = _h_precision_score(y_true, y_pred, class_hierarchy, root=root)\n",
    "    hR = _h_recall_score(y_true, y_pred, class_hierarchy, root=root)\n",
    "    if hP ==hR==0:\n",
    "        return (0)\n",
    "    return (1. + beta ** 2.) * hP * hR / (beta ** 2. * hP + hR)\n",
    "    \n",
    "def _fill_ancestors(y, graph, root, copy=True):\n",
    "    y_ = y.copy() if copy else y\n",
    "    paths = all_pairs_shortest_path_length(graph.reverse(copy=False))\n",
    "    for target, distances in paths:\n",
    "        if target == root:\n",
    "            continue\n",
    "        ix_rows = np.where(y[:, target] > 0)[0]\n",
    "        ancestors = list(filter(lambda x: x != ROOT,distances.keys()))\n",
    "        y_[tuple(np.meshgrid(ix_rows, ancestors))] = 1\n",
    "    graph.reverse(copy=False)\n",
    "    return y_\n",
    "def _h_recall_score(y_true, y_pred, class_hierarchy, root=ROOT):\n",
    "    y_true_ = _fill_ancestors(y_true, graph=class_hierarchy, root=root)\n",
    "    y_pred_ = _fill_ancestors(y_pred, graph=class_hierarchy, root=root)\n",
    "\n",
    "    ix = np.where((y_true_ != 0) & (y_pred_ != 0))\n",
    "\n",
    "    true_positives = len(ix[0])\n",
    "    all_positives = np.count_nonzero(y_true_)\n",
    "    if all_positives==0:\n",
    "        return 0.000000001\n",
    "\n",
    "    \n",
    "    return true_positives / all_positives\n",
    "\n",
    "def _h_precision_score(y_true, y_pred, class_hierarchy, root=ROOT):\n",
    "    y_true_ = _fill_ancestors(y_true, graph=class_hierarchy, root=root)\n",
    "    y_pred_ = _fill_ancestors(y_pred, graph=class_hierarchy, root=root)\n",
    "\n",
    "    ix = np.where((y_true_ != 0) & (y_pred_ != 0))\n",
    "\n",
    "    true_positives = len(ix[0])\n",
    "    all_results = np.count_nonzero(y_pred_)\n",
    "    if all_results==0:\n",
    "        return 0.000000001\n",
    "    \n",
    "    return true_positives / all_results\n",
    "def read_classes(file_path):\n",
    "  CLASSES = []\n",
    "  with open(file_path) as f:\n",
    "    for label in f.readlines():\n",
    "      label = label.strip()\n",
    "      if label:\n",
    "        CLASSES.append(label)\n",
    "  return CLASSES\n",
    "\n",
    "def check_format(file_path):\n",
    "  _classes = get_all_classes_from_graph(G)\n",
    "  if not os.path.exists(file_path):\n",
    "    logging.error(\"File doesnt exists: {}\".format(file_path))\n",
    "    return False\n",
    "  submmission = ''\n",
    "  try:\n",
    "    with open(file_path, encoding='utf-8') as p:\n",
    "      submission = json.load(p)\n",
    "  except:\n",
    "    logging.error(\"File is not a valid json file: {}\".format(file_path))\n",
    "    return False\n",
    "  for i, obj in enumerate(submission):\n",
    "    for key in KEYS:\n",
    "      if key not in obj:\n",
    "        logging.error(\"Missing entry in {}:{}\".format(file_path, i))\n",
    "        return False\n",
    "  for label in list(obj['labels']):\n",
    "       if label not in _classes:\n",
    "         print(label)\n",
    "         logging.error(\"Unknown Label in {}:{}\".format(file_path, i))\n",
    "         return False\n",
    "  return True\n",
    "\n",
    "def _read_gold_and_pred(pred_fpath, gold_fpath):\n",
    "  \"\"\"\n",
    "  Read gold and predicted data.\n",
    "  :param pred_fpath: a json file with predictions, \n",
    "  :param gold_fpath: the original annotated gold file.\n",
    "  :return: {id:pred_labels} dict; {id:gold_labels} dict\n",
    "  \"\"\"\n",
    "\n",
    "  gold_labels = {}\n",
    "  with open(gold_fpath, encoding='utf-8') as gold_f:\n",
    "    gold = json.load(gold_f)\n",
    "    for obj in gold:\n",
    "      gold_labels[obj['id']] = obj['labels']\n",
    "\n",
    "  pred_labels = {}\n",
    "  with open(pred_fpath, encoding='utf-8') as pred_f:\n",
    "    pred = json.load(pred_f)\n",
    "    for obj in pred:\n",
    "      pred_labels[obj['id']] = obj['labels']\n",
    "\n",
    "  if set(gold_labels.keys()) != set(pred_labels.keys()):\n",
    "      logger.error('There are either missing or added examples to the prediction file. Make sure you only have the gold examples in the prediction file.')\n",
    "      raise ValueError('There are either missing or added examples to the prediction file. Make sure you only have the gold examples in the prediction file.')\n",
    "  \n",
    "  return pred_labels, gold_labels\n",
    "\n",
    "def evaluate_h(pred_labels, gold_labels):\n",
    "    #pred_labels, gold_labels = _read_gold_and_pred(pred_file, gold_file)\n",
    "  \n",
    "    gold = []\n",
    "    pred = []\n",
    "    for id in gold_labels:\n",
    "        gold.append(gold_labels[id])\n",
    "        pred.append(pred_labels[id])\n",
    "    with multi_labeled(gold, pred, G) as (gold_, pred_, graph_):\n",
    "        return  _h_precision_score(gold_, pred_,graph_), _h_recall_score(gold_, pred_,graph_), _h_fbeta_score(gold_, pred_,graph_)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24204ccd",
   "metadata": {},
   "source": [
    "# Imports "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "180e008c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import deque\n",
    "import random\n",
    "import copy\n",
    "import json\n",
    "\n",
    "import torch\n",
    "import sklearn.metrics as metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from transformers import AutoTokenizer, BertModel, BertForSequenceClassification, RobertaForSequenceClassification,  AlbertForSequenceClassification\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau, LambdaLR\n",
    "from torch.optim import AdamW\n",
    "\n",
    "from tqdm.notebook import tqdm, trange\n",
    "\n",
    "random_seed = 0\n",
    "torch.manual_seed(random_seed)\n",
    "random.seed(random_seed)\n",
    "np.random.seed(random_seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4f86c4ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "TECHNIQUES_RAW = \"\"\"Repetition\n",
    "Obfuscation, Intentional vagueness, Confusion\n",
    "Slogans\n",
    "Bandwagon\n",
    "Appeal to authority\n",
    "Flag-waving\n",
    "Appeal to fear/prejudice\n",
    "Causal Oversimplification\n",
    "Black-and-white Fallacy/Dictatorship\n",
    "Thought-terminating cliché\n",
    "Misrepresentation of Someone's Position (Straw Man)\n",
    "Presenting Irrelevant Data (Red Herring)\n",
    "Whataboutism\n",
    "Glittering generalities (Virtue)\n",
    "Doubt\n",
    "Name calling/Labeling\n",
    "Smears\n",
    "Reductio ad hitlerum\n",
    "Exaggeration/Minimisation\n",
    "Loaded Language\"\"\"\n",
    "\n",
    "TECHNIQUES_OTHER = \"\"\"\n",
    "Logos\n",
    "Reasoning\n",
    "Justification\n",
    "Simplification\n",
    "Distraction\n",
    "Ethos\n",
    "Ad Hominem\n",
    "Pathos\"\"\"\n",
    "\n",
    "\n",
    "TECHNIQUES_RAW = TECHNIQUES_RAW+TECHNIQUES_OTHER\n",
    "\n",
    "\n",
    "#TECHNIQUES_RAW = \"\"\"Presenting Irrelevant Data (Red Herring)\"\"\"\n",
    "TECHNIQUES = TECHNIQUES_RAW.split('\\n')\n",
    "#TECHNIQUES = ['Logos', 'Pathos', 'Ethos']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e2bd4382",
   "metadata": {},
   "outputs": [],
   "source": [
    "LOGOS_all = [\"Repetition\", \"Obfuscation, Intentional vagueness, Confusion\", \"Reasoning\", \"Justification\"\n",
    "        ,\"Slogans\", \"Bandwagon\", \"Appeal to authority\", \"Flag-waving\", \"Appeal to fear/prejudice\"\n",
    "        ,\"Simplification\", \"Causal Oversimplification\", \"Black-and-white Fallacy/Dictatorship\", \"Thought-terminating cliché\"\n",
    "        ,\"Distraction\",\"Misrepresentation of Someone's Position (Straw Man)\", \"Presenting Irrelevant Data (Red Herring)\", \"Whataboutism\"]\n",
    "ETHOS_all = [\"Appeal to authority\", \"Glittering generalities (Virtue)\", \"Bandwagon\", \"Ad Hominem\", \"Transfer\"\n",
    "        ,\"Doubt\", \"Name calling/Labeling\", \"Smears\", \"Reductio ad hitlerum\", \"Whataboutism\"]\n",
    "PATHOS_all = [\"Exaggeration/Minimisation\", \"Loaded Language\", \"Appeal to (Strong) Emotions\", \"Appeal to fear/prejudice\", \"Flag-waving\", \"Transfer\"]\n",
    "JUSTIFICATION_all = [\"Slogans\", \"Bandwagon\", \"Appeal to authority\", \"Flag-waving\", \"Appeal to fear/prejudice\"]\n",
    "REASONING_all = [\"Simplification\", \"Distraction\" , \"Causal Oversimplification\", \"Black-and-white Fallacy/Dictatorship\", \"Thought-terminating cliché\"\n",
    "                ,\"Misrepresentation of Someone's Position (Straw Man)\", \"Presenting Irrelevant Data (Red Herring)\", \"Whataboutism\"]\n",
    "SIMPLIFICATION_all = [\"Causal Oversimplification\", \"Black-and-white Fallacy/Dictatorship\", \"Thought-terminating cliché\"]\n",
    "DISTRACTION_all = [\"Misrepresentation of Someone's Position (Straw Man)\", \"Presenting Irrelevant Data (Red Herring)\", \"Whataboutism\"]\n",
    "ADHOMINEM_all = [\"Doubt\", \"Name calling/Labeling\", \"Smears\", \"Reductio ad hitlerum\", \"Whataboutism\"]\n",
    "\n",
    "SUPERCLASSES_all = [LOGOS_all, ETHOS_all, PATHOS_all, JUSTIFICATION_all, REASONING_all, SIMPLIFICATION_all, DISTRACTION_all, ADHOMINEM_all]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4646839d",
   "metadata": {},
   "outputs": [],
   "source": [
    "LOGOS = [\"Repetition\", \"Obfuscation, Intentional vagueness, Confusion\", \"Reasoning\", \"Justification\"\n",
    "        ,\"Slogans\", \"Bandwagon\", \"Appeal to authority\", \"Flag-waving\", \"Appeal to fear/prejudice\"\n",
    "        ,\"Simplification\", \"Causal Oversimplification\", \"Black-and-white Fallacy/Dictatorship\", \"Thought-terminating cliché\"\n",
    "        ,\"Distraction\",\"Misrepresentation of Someone's Position (Straw Man)\", \"Presenting Irrelevant Data (Red Herring)\", \"Whataboutism\"]\n",
    "ETHOS = [\"Appeal to authority\", \"Glittering generalities (Virtue)\", \"Bandwagon\", \"Ad Hominem\", \"Transfer\"\n",
    "        ,\"Doubt\", \"Name calling/Labeling\", \"Smears\", \"Reductio ad hitlerum\", \"Whataboutism\"]\n",
    "PATHOS = [\"Exaggeration/Minimisation\", \"Loaded Language\", \"Appeal to (Strong) Emotions\", \"Appeal to fear/prejudice\", \"Flag-waving\", \"Transfer\"]\n",
    "JUSTIFICATION = [\"Slogans\", \"Bandwagon\", \"Appeal to authority\", \"Flag-waving\", \"Appeal to fear/prejudice\"]\n",
    "REASONING = [\"Simplification\", \"Distraction\"]\n",
    "SIMPLIFICATION = [\"Causal Oversimplification\", \"Black-and-white Fallacy/Dictatorship\", \"Thought-terminating cliché\"]\n",
    "DISTRACTION = [\"Misrepresentation of Someone's Position (Straw Man)\", \"Presenting Irrelevant Data (Red Herring)\", \"Whataboutism\"]\n",
    "ADHOMINEM = [\"Doubt\", \"Name calling/Labeling\", \"Smears\", \"Reductio ad hitlerum\", \"Whataboutism\"]\n",
    "\n",
    "SUPERCLASSES = [LOGOS, ETHOS, PATHOS, JUSTIFICATION, REASONING, SIMPLIFICATION, DISTRACTION, ADHOMINEM]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dc58b419",
   "metadata": {},
   "outputs": [],
   "source": [
    "CONVERSION_PTC_SEMEVAL = {}\n",
    "CONVERSION_PTC_SEMEVAL[\"Exaggeration,Minimisation\"] = \"Exaggeration/Minimisation\"\n",
    "CONVERSION_PTC_SEMEVAL[\"Thought-terminating_Cliches\"] = \"Thought-terminating cliché\"\n",
    "CONVERSION_PTC_SEMEVAL[\"Flag-Waving\"] = \"Flag-waving\"\n",
    "CONVERSION_PTC_SEMEVAL[\"Appeal_to_fear-prejudice\"] = \"Appeal to fear/prejudice\"\n",
    "CONVERSION_PTC_SEMEVAL[\"Causal_Oversimplification\"] = \"Causal Oversimplification\"\n",
    "CONVERSION_PTC_SEMEVAL[\"Appeal_to_Authority\"] = \"Appeal to authority\"\n",
    "CONVERSION_PTC_SEMEVAL[\"Whataboutism,Straw_Men,Red_Herring\"] = \"Whataboutism\" #here\n",
    "CONVERSION_PTC_SEMEVAL[\"Bandwagon,Reductio_ad_hitlerum\"] = \"Reductio ad hitlerum\" #here\n",
    "CONVERSION_PTC_SEMEVAL[\"Loaded_Language\"] = \"Loaded Language\"\n",
    "CONVERSION_PTC_SEMEVAL[\"Black-and-White_Fallacy\"] = \"Black-and-white Fallacy/Dictatorship\"\n",
    "CONVERSION_PTC_SEMEVAL[\"Name_Calling,Labeling\"] = \"Name calling/Labeling\"\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3657466",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3dbda0b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def transformData_3way(train):\n",
    "    #0: Logos, 1: Pathos, 2: Ethos\n",
    "    train_h = []\n",
    "    for i in range(0, len(train)):\n",
    "        d = {}\n",
    "        t = train[i]\n",
    "        L = t['labels']\n",
    "        L2 = []\n",
    "        for tech in L:\n",
    "            if tech in PATHOS:\n",
    "                L2.append(\"Pathos\")\n",
    "            if tech in LOGOS:\n",
    "                L2.append(\"Logos\")\n",
    "            if tech in ETHOS:\n",
    "                L2.append(\"Ethos\")\n",
    "        L2 = list(set(L2))\n",
    "        d['labels'] = L2\n",
    "        d['text'] = t['text']\n",
    "        d['id'] = t['id']\n",
    "        train_h.append(d)\n",
    "    return train_h\n",
    "\n",
    "def transformData_1class(train):\n",
    "    #0: Logos, 1: Pathos, 2: Ethos\n",
    "    train_h = []\n",
    "    for i in range(0, len(train)):\n",
    "        d = {}\n",
    "        t = train[i]\n",
    "        L = t['labels']\n",
    "        L2 = []\n",
    "        if TECHNIQUES[0] in L:\n",
    "            L2.append(TECHNIQUES[0])\n",
    "        L2 = list(set(L2))\n",
    "        d['labels'] = L2\n",
    "        d['text'] = t['text']\n",
    "        d['id'] = t['id']\n",
    "        train_h.append(d)\n",
    "    return train_h\n",
    "\n",
    "\n",
    "def addSuperClasses(train):\n",
    "    train_h = []\n",
    "    for i in range(0, len(train)):\n",
    "        d = {}\n",
    "        t = train[i]\n",
    "        L = t['labels']\n",
    "        L2 = []\n",
    "        for tech in L:\n",
    "            L2.append(tech)\n",
    "            if tech in PATHOS:\n",
    "                L2.append(\"Pathos\")\n",
    "            if tech in LOGOS:\n",
    "                L2.append(\"Logos\")\n",
    "            if tech in ETHOS:\n",
    "                L2.append(\"Ethos\")\n",
    "            if tech in JUSTIFICATION:\n",
    "                L2.append(\"Justification\")\n",
    "                L2.append(\"Logos\")\n",
    "            if tech in REASONING:\n",
    "                L2.append(\"Reasoning\")\n",
    "                L2.append(\"Logos\")\n",
    "            if tech in SIMPLIFICATION:\n",
    "                L2.append(\"Simplification\")\n",
    "                L2.append(\"Reasoning\")\n",
    "                L2.append(\"Logos\")\n",
    "            if tech in DISTRACTION:\n",
    "                L2.append('Distraction')\n",
    "                L2.append(\"Reasoning\")\n",
    "                L2.append(\"Logos\")\n",
    "            if tech in ADHOMINEM:\n",
    "                L2.append('Ad Hominem')\n",
    "                L2.append(\"Ethos\")\n",
    "        L2 = list(set(L2))\n",
    "        d['labels'] = L2\n",
    "        d['text'] = t['text']\n",
    "        d['id'] = t['id']\n",
    "        train_h.append(d)\n",
    "    return train_h\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0a8f9ef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/path/to/semeval/data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9066d386",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'path' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/opt/tmp/ipykernel_2402547/4190092374.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'train.json'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mtrain\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'validation.json'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mvalidation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'dev_gold_labels/dev_subtask1_en.json'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'path' is not defined"
     ]
    }
   ],
   "source": [
    "with open(path+'train.json', 'r') as f:\n",
    "    train = json.load(f)\n",
    "with open(path+'validation.json', 'r') as f:\n",
    "    validation = json.load(f)\n",
    "with open(path+'dev_gold_labels/dev_subtask1_en.json', 'r') as f:\n",
    "    dev = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "23927bc0",
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(path+'2021_data/training_set_task1.txt', 'r') as f:\n",
    "    old_train = json.load(f)\n",
    "with open(path+'2021_data/test_set_task1.txt', 'r') as f:\n",
    "    old_test = json.load(f)\n",
    "with open(path+'2021_data/dev_set_task1.txt', 'r') as f:\n",
    "    old_dev = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c8838963",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ptc = pd.read_csv(path+'PTC/datasets/train_sentences.csv')\n",
    "dev_ptc = pd.read_csv(path+'PTC/datasets/dev_sentences.csv')\n",
    "\n",
    "ptc_data = pd.concat([train_ptc, dev_ptc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "36be525d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sents = ptc_data['sentences'].tolist()\n",
    "props = ptc_data['propaganda'].tolist()\n",
    "\n",
    "very_old_train = []\n",
    "for i in range(0, len(props)):\n",
    "    s = sents[i]\n",
    "    p = props[i]\n",
    "    p = p.replace(\"\\'\", '\\\"')\n",
    "    p = json.loads(p)\n",
    "    if p !=[]:\n",
    "        d = {}\n",
    "        pp = []\n",
    "        for propaganda in p:\n",
    "            p2 = propaganda\n",
    "            if p2 not in TECHNIQUES:\n",
    "                p2 = CONVERSION_PTC_SEMEVAL[propaganda]\n",
    "            pp.append(p2)\n",
    "        d['labels'] = pp\n",
    "        d['text'] = s\n",
    "        very_old_train.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "581b1940",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path+'0_augmented50.json', 'r') as f:\n",
    "    red_herring = json.load(f)\n",
    "with open(path+'1_augmented50.json', 'r') as f:\n",
    "    obfuscation = json.load(f)\n",
    "with open(path+'2_augmented50.json', 'r') as f:\n",
    "    strawman = json.load(f)\n",
    "with open(path+'3_augmented50.json', 'r') as f:\n",
    "    thought = json.load(f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "3ebb12b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "augmented = []\n",
    "problematic_conspiracies = [\n",
    "    \"Presenting Irrelevant Data (Red Herring)\",\n",
    "    \"Obfuscation, Intentional vagueness, Confusion\",\n",
    "    \"Misrepresentation of Someone's Position (Straw Man)\",\n",
    "    \"Thought-terminating cliché\"\n",
    "]\n",
    "\n",
    "for data_augmented in [red_herring, obfuscation, strawman, thought]:\n",
    "    for text in data_augmented:\n",
    "        d = {}\n",
    "        d['text'] = text\n",
    "        d['labels'] = [problematic_conspiracies[[red_herring, obfuscation, strawman, thought].index(data_augmented)]]\n",
    "        augmented.append(d)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "188173f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8902"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train+old_train+old_test+old_dev#+very_old_train\n",
    "len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d60bc301",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = addSuperClasses(train)\n",
    "validation = addSuperClasses(validation)\n",
    "dev = addSuperClasses(dev)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b02b2f45",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_text = []\n",
    "train_labels = []\n",
    "train_labels_binary = []\n",
    "for i in range(0, len(train)):\n",
    "    doc = train[i]\n",
    "    train_text.append(doc['text'])\n",
    "    \n",
    "    l = doc['labels']\n",
    "    L = [0 for i in range(0, len(TECHNIQUES))]\n",
    "    for t in l:\n",
    "        L[TECHNIQUES.index(t)] = 1\n",
    "    train_labels.append(L)\n",
    "    train_labels_binary.append(1 if sum(L)>0 else 0)\n",
    "    \n",
    "validation_text = []\n",
    "validation_labels = []\n",
    "validation_labels_binary = []\n",
    "for i in range(0, len(validation)):\n",
    "    doc = validation[i]\n",
    "    validation_text.append(doc['text'])\n",
    "    \n",
    "    l = doc['labels']\n",
    "    L = [0 for i in range(0, len(TECHNIQUES))]\n",
    "    for t in l:\n",
    "        L[TECHNIQUES.index(t)] = 1\n",
    "    validation_labels.append(L)\n",
    "    validation_labels_binary.append(1 if sum(L)>0 else 0)\n",
    "\n",
    "dev_text = []\n",
    "dev_labels = []\n",
    "dev_labels_binary = []\n",
    "for i in range(0, len(dev)):\n",
    "    doc = dev[i]\n",
    "    dev_text.append(doc['text'])\n",
    "    \n",
    "    l = doc['labels']\n",
    "    L = [0 for i in range(0, len(TECHNIQUES))]\n",
    "    for t in l:\n",
    "        L[TECHNIQUES.index(t)] = 1\n",
    "    dev_labels.append(L)\n",
    "    dev_labels_binary.append(1 if sum(L)>0 else 0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "26047e23",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[  1.0081, 124.2344]], device='cuda:0')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_tmp = [0 for i in range(0, len(TECHNIQUES))]\n",
    "for i in range(0, len(TECHNIQUES)):\n",
    "    for j in range(0, len(train_labels)):\n",
    "        if train_labels[j][i]>0:\n",
    "            weights_tmp[i]+=1\n",
    "\n",
    "weights_techniques = torch.Tensor([[len(train_labels)/(len(train_labels)-w), len(train_labels)/w] for w in weights_tmp]).to('cuda')\n",
    "weights_techniques"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "fb94856e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "375"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')\n",
    "#tokenizer = AutoTokenizer.from_pretrained('albert-base-v2')\n",
    "\n",
    "tokenized_input = tokenizer(train_text)\n",
    "\n",
    "m = 0\n",
    "for tokens in tokenized_input['input_ids']:\n",
    "    if len(tokens)>m:\n",
    "        m=len(tokens)\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2c706c24",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/tmp/ipykernel_2636470/2841440615.py:20: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  train_token_type_ids = torch.tensor(train_token_type_ids)\n",
      "/opt/tmp/ipykernel_2636470/2841440615.py:26: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  validation_token_type_ids = torch.tensor(validation_token_type_ids)\n",
      "/opt/tmp/ipykernel_2636470/2841440615.py:32: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  dev_token_type_ids = torch.tensor(dev_token_type_ids)\n"
     ]
    }
   ],
   "source": [
    "MAX_LEN = 256 #128 < m some texts will be truncated\n",
    "\n",
    "tokenized_train = tokenizer(train_text, max_length=MAX_LEN, padding='max_length', truncation=True)\n",
    "tokenized_validation = tokenizer(validation_text, max_length=MAX_LEN, padding='max_length', truncation=True)\n",
    "tokenized_dev = tokenizer(dev_text, max_length=MAX_LEN, padding='max_length', truncation=True)\n",
    "    \n",
    "train_input_ids, train_token_type_ids, train_attention_mask = tokenized_train['input_ids'], tokenized_train['token_type_ids'], tokenized_train['attention_mask']\n",
    "validation_input_ids, validation_token_type_ids, validation_attention_mask = tokenized_validation['input_ids'], tokenized_validation['token_type_ids'], tokenized_validation['attention_mask']\n",
    "dev_input_ids, dev_token_type_ids, dev_attention_mask = tokenized_dev['input_ids'], tokenized_dev['token_type_ids'], tokenized_dev['attention_mask']\n",
    "\n",
    "train_token_type_ids = torch.tensor(train_token_type_ids)\n",
    "validation_token_type_ids = torch.tensor(validation_token_type_ids)\n",
    "dev_token_type_ids = torch.tensor(dev_token_type_ids)\n",
    "\n",
    "# Convert to torch tensor\n",
    "train_input_ids = torch.tensor(train_input_ids)\n",
    "train_labels = torch.tensor(train_labels)\n",
    "train_labels_binary = torch.tensor(train_labels_binary)\n",
    "train_attention_mask = torch.tensor(train_attention_mask)\n",
    "train_token_type_ids = torch.tensor(train_token_type_ids)\n",
    "\n",
    "validation_input_ids = torch.tensor(validation_input_ids)\n",
    "validation_labels = torch.tensor(validation_labels)\n",
    "validation_labels_binary = torch.tensor(validation_labels_binary)\n",
    "validation_attention_mask = torch.tensor(validation_attention_mask)\n",
    "validation_token_type_ids = torch.tensor(validation_token_type_ids)\n",
    "\n",
    "dev_input_ids = torch.tensor(dev_input_ids)\n",
    "dev_labels = torch.tensor(dev_labels)\n",
    "dev_labels_binary = torch.tensor(dev_labels_binary)\n",
    "dev_attention_mask = torch.tensor(dev_attention_mask)\n",
    "dev_token_type_ids = torch.tensor(dev_token_type_ids)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ff5e407c",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 16 #\n",
    "\n",
    "train_data = TensorDataset(train_input_ids, train_attention_mask, train_labels, train_token_type_ids)\n",
    "validation_data = TensorDataset(validation_input_ids, validation_attention_mask, validation_labels, validation_token_type_ids)\n",
    "dev_data = TensorDataset(dev_input_ids, dev_attention_mask, dev_labels, dev_token_type_ids)\n",
    "\n",
    "    \n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
    "\n",
    "validation_sampler = SequentialSampler(validation_data)\n",
    "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)\n",
    "\n",
    "dev_sampler = SequentialSampler(dev_data)\n",
    "dev_dataloader = DataLoader(dev_data, sampler=dev_sampler, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ecabe4d",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4f88a8f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class bert(nn.Module):\n",
    "    \n",
    "    def __init__(self, n_classes):\n",
    "        super().__init__()\n",
    "        self.n_classes = n_classes\n",
    "        self.bert = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=n_classes)\n",
    "        #self.bert = RobertaForSequenceClassification.from_pretrained(\"roberta-base\", num_labels=n_classes)\n",
    "        \n",
    "    def forward(self, input_ids, token_type_ids, input_mask):\n",
    "        outputs = self.bert(input_ids = input_ids, token_type_ids = token_type_ids, attention_mask = input_mask)\n",
    "\n",
    "        logits = outputs.logits\n",
    "        \n",
    "        return logits  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "67f62c66",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "bert(\n",
       "  (bert): BertForSequenceClassification(\n",
       "    (bert): BertModel(\n",
       "      (embeddings): BertEmbeddings(\n",
       "        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "        (position_embeddings): Embedding(512, 768)\n",
       "        (token_type_embeddings): Embedding(2, 768)\n",
       "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (encoder): BertEncoder(\n",
       "        (layer): ModuleList(\n",
       "          (0): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (1): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (2): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (3): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (4): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (5): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (6): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (7): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (8): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (9): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (10): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (11): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (pooler): BertPooler(\n",
       "        (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (activation): Tanh()\n",
       "      )\n",
       "    )\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "    (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device=\"cuda\"\n",
    "\n",
    "model = bert(len(TECHNIQUES)*2) # *2 for CE loss\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "f814ce05",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Freeze layers\n",
    "# for param in model.bert.bert.encoder.layer[:3].parameters():\n",
    "#     param.requires_grad = False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "cf2ef5bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#optimizer_grouped_parameters\n",
    "optimizer = AdamW(model.parameters(),\n",
    "                  lr=7e-5,\n",
    "                  #lr=5e-6,\n",
    "                  weight_decay = 0.01)\n",
    "\n",
    "scheduler = ReduceLROnPlateau(optimizer, patience=4, factor=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "8d82d34e",
   "metadata": {},
   "outputs": [],
   "source": [
    "criterions = []\n",
    "\n",
    "for i in range(0, len(TECHNIQUES)):\n",
    "    criterions.append(nn.CrossEntropyLoss(weight = weights_techniques[i]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "0a3074e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.BCEWithLogitsLoss(pos_weight=weights_techniques[:,1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26a16bf9",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "69e58923",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_numbers_to_name(predictions_sep):\n",
    "    PREDS = [[predictions_sep[i][k] for i in range(0, len(TECHNIQUES))] for k in range(0, len(predictions_sep[0]))]\n",
    "    PREDS_NAMES = []\n",
    "    for i in range(0, len(PREDS)):\n",
    "        P = []\n",
    "        for k in range(0, len(TECHNIQUES)):\n",
    "            p = PREDS[i][k]\n",
    "            if p==1:\n",
    "                P.append(TECHNIQUES[k])\n",
    "        PREDS_NAMES.append(P)\n",
    "        \n",
    "    return (PREDS_NAMES)\n",
    "\n",
    "def evaluate_preds(predictions_sep, dev):\n",
    "    PREDS_NAMES = convert_numbers_to_name(predictions_sep)\n",
    "    \n",
    "    prediction_dict = {}\n",
    "    ground_truth_dict = {}\n",
    "    for i in range(0, len(dev)):\n",
    "        prediction_dict[dev[i]['id']] = PREDS_NAMES[i]\n",
    "        ground_truth_dict[dev[i]['id']] = dev[i]['labels']\n",
    "    \n",
    "    p, r, f = evaluate_h(prediction_dict, ground_truth_dict)\n",
    "    return (p, r, f)\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d79b4a3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.03774619102478027,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 40,
       "postfix": null,
       "prefix": "",
       "rate": null,
       "total": 15,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "862d3c48ee774be98737c65211bd273f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/15 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting epoch  0\n",
      "Train loss: 0.3923068821795146\n",
      "\t Eval loss: 0.3691267544636503\n",
      "\t Eval F1H: 0.0, PREH: 0.0, RECH: 0.0\n",
      "\t Eval F1s: [0.0]\n",
      "---------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "Starting epoch  1\n",
      "Train loss: 0.3708395712692253\n",
      "\t Eval loss: 0.3494117090012878\n",
      "\t Eval F1H: 0.0, PREH: 0.0, RECH: 0.0\n",
      "\t Eval F1s: [0.0]\n",
      "---------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "Starting epoch  2\n",
      "Train loss: 0.3609117641820154\n",
      "\t Eval loss: 0.36308928159996867\n",
      "\t Eval F1H: 0.0, PREH: 0.0, RECH: 0.0\n",
      "\t Eval F1s: [0.0]\n",
      "---------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "Starting epoch  3\n",
      "Train loss: 0.3476922157139965\n",
      "\t Eval loss: 0.3867275263182819\n",
      "\t Eval F1H: 0.0, PREH: 0.0, RECH: 0.0\n",
      "\t Eval F1s: [0.0]\n",
      "---------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "Starting epoch  4\n",
      "Train loss: 0.346532984301298\n",
      "\t Eval loss: 0.3903538975864649\n",
      "\t Eval F1H: 0.0, PREH: 0.0, RECH: 0.0\n",
      "\t Eval F1s: [0.0]\n",
      "---------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "Starting epoch  5\n"
     ]
    }
   ],
   "source": [
    "epochs = 15\n",
    "\n",
    "best_loss = 999\n",
    "best_state_dict = model.state_dict()\n",
    "best_epoch = 0\n",
    "METRICS = []\n",
    "best_f1h = 0\n",
    "sig = nn.Sigmoid()\n",
    "\n",
    "for e in trange(0, epochs):\n",
    "\n",
    "    # Training\n",
    "    print('Starting epoch ', e)\n",
    "    model.train()\n",
    "    \n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "\n",
    "        b_input_ids, b_input_mask, b_labels, b_token_type_ids = batch\n",
    "        \n",
    "        b_labels = b_labels.float()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logits = model(b_input_ids, b_token_type_ids, b_input_mask) #.logits\n",
    "        \n",
    "        losses = []\n",
    "        for i in range(0, len(TECHNIQUES)):\n",
    "            logits_i = logits[:,2*i:2*i+2]\n",
    "            labels_i = b_labels[:, i].long()\n",
    "            loss_i = criterions[i](logits_i, labels_i)\n",
    "            losses.append(loss_i)\n",
    "        loss = sum(losses)\n",
    "        \n",
    "        #loss = criterion(logits, b_labels)\n",
    "        \n",
    "        \n",
    "        #loss = criterion(logits, b_labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += b_input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "\n",
    "    print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
    "    \n",
    "    \n",
    "    # Testing\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    tweets_test = []\n",
    "    \n",
    "    predictions_sep = [[], [], [], [], [], [], [], [], [],[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]\n",
    "\n",
    "    labels_sep = [[], [], [], [], [], [], [], [], [],[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]\n",
    "    \n",
    "    eval_loss = 0\n",
    "    steps=0\n",
    "    for step, batch in enumerate(validation_dataloader):\n",
    "\n",
    "        # Add batch to GPU\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "\n",
    "        b_input_ids, b_input_mask, b_labels, b_token_type_ids = batch\n",
    "            \n",
    "        b_labels = b_labels.float()\n",
    "        \n",
    "        with torch.no_grad():\n",
    "\n",
    "            logits = model(b_input_ids, b_token_type_ids, b_input_mask)\n",
    "            losses = []\n",
    "            for i in range(0, len(TECHNIQUES)):\n",
    "                logits_i = logits[:,2*i:2*i+2]\n",
    "                labels_i = b_labels[:, i].long()\n",
    "                loss_i = criterions[i](logits_i, labels_i)\n",
    "                losses.append(loss_i)\n",
    "            loss = sum(losses)\n",
    "            #loss = criterion(logits, b_labels.long())\n",
    "            #loss = criterion(logits, b_labels)\n",
    "\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        ground_truth = b_labels.detach().cpu().numpy()\n",
    "        \n",
    "        steps+=1\n",
    "        eval_loss+=loss.detach().item()\n",
    "                \n",
    "        for i in range(0, len(TECHNIQUES)):\n",
    "            for p in logits:\n",
    "                p_i = p[2*i:2*i+2]\n",
    "                pred = np.argmax(p_i)\n",
    "                predictions_sep[i].append(pred)\n",
    "            #for p in logits:\n",
    "            #    p_sig = sig(torch.Tensor(p))\n",
    "            #    p_01 = (p_sig>0.5).int()\n",
    "            #    predictions_sep[i].append(p_01[i].item())\n",
    "                \n",
    "            for l in ground_truth:\n",
    "                labels_sep[i].append(l[i])\n",
    "        #labels_sep[0].extend(b_labels.int().cpu().numpy())\n",
    "        #predictions_sep[0].extend(logits.argmax(1))\n",
    "        \n",
    "    #scheduler.step(e)          \n",
    "    scheduler.step(eval_loss/steps)\n",
    "    LOSS = eval_loss/steps\n",
    "    \n",
    "    precision, recall, f1_h = evaluate_preds(predictions_sep, validation)\n",
    "    F1s = []\n",
    "    for i in range(0, len(TECHNIQUES)):\n",
    "        F1s.append(round(metrics.f1_score(labels_sep[i], predictions_sep[i]), 3))\n",
    "    #precision, recall, f1_h, support =  metrics.precision_recall_fscore_support(labels_sep[0], predictions_sep[0], average='macro')\n",
    "    METRICS.append([precision, recall, f1_h, F1s])\n",
    "    \n",
    "    if f1_h> best_f1h:\n",
    "        best_loss = LOSS\n",
    "        best_state_dict = copy.deepcopy(model.state_dict())\n",
    "        best_epoch = e\n",
    "        best_f1h = f1_h\n",
    "    \n",
    "    print(\"\\t Eval loss: {}\".format(LOSS))\n",
    "    print(\"\\t Eval F1H: {}, PREH: {}, RECH: {}\".format(round(f1_h, 3), round(precision, 3), round(recall, 3)))\n",
    "    print(\"\\t Eval F1s: {}\".format(F1s))\n",
    "    print(\"---\"*25)\n",
    "    print(\"\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8726b91b",
   "metadata": {},
   "outputs": [],
   "source": [
    "[m[2] for m in METRICS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2706ac71",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(best_state_dict, path+'best_model.pth')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7532107",
   "metadata": {},
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9b4c9875",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " task1_AlbertW_merged_all_data_e9_0.598.pth\r\n",
      " task1_ALBERTW_merged_data__e13_0.551.pth\r\n",
      " task1_bert-base-uncased-finetuned-hateful-memeW_merged_all_data_e9_0.618.pth\r\n",
      " task1_BERT_casedW_merged_all_data_e8_0.578.pth\r\n",
      " task1_BERT_casedW_merged_data__e9_0.569.pth\r\n",
      " task1_BERT_merged_all_data+PTC_e6_0.566.pth\r\n",
      " task1_BERT_merged_data_BCE_unweighted_e9_0.58.pth\r\n",
      " task1_BERT_merged_data_BCE_weighted_e9_0.603.pth\r\n",
      " task1_BERT_merged_data_BCE_weighted_epoch15_lr3e5_e9_0.605.pth\r\n",
      " task1_BERT_merged_data_e8_0.627.pth\r\n",
      " task1_BERT_merged_data_inbetween_weights_e9_0.623.pth\r\n",
      " task1_BERT_merged_data_more_weights_e11_0.613.pth\r\n",
      " task1_BERT_merged_data_more_weights_freeze_3_e9_0.625.pth\r\n",
      " task1_BERT_merged_data+ptc_BCE_weighted_epoch15_lr3e5_no_shedule_e12_0.611.pth\r\n",
      " task1_BERT_merged_data+ptc_BCE_weighted_epoch15_lr3e5_no_shedule_e4_0.4.pth\r\n",
      " task1_BERT_merged_data_seed42_e5_0.589.pth\r\n",
      " task1_BERTW_34_weights_only_positive_merged_all_data_e9_0.614.pth\r\n",
      " task1_BERTW_3way_merged_all_data_e1_0.734.pth\r\n",
      " task1_BERTW_all_PTC_e5_0.067.pth\r\n",
      " task1_BERTW_bert_token_e9_0.609.pth\r\n",
      " task1_BERTW_binary_weights_merged_all_data_e6_0.761.pth\r\n",
      " task1_BERTW_double_weights_only_positive_merged_all_data+PTC_e6_0.61.pth\r\n",
      " task1_BERTW_e9_0.609.pth\r\n",
      " task1_BERTW_merged_all_data_augmented_e1_0.623.pth\r\n",
      " task1_BERTW_merged_all_data_focal_loss_e6_0.565.pth\r\n",
      " task1_BERTW_merged_all_data_focal_loss_kornia_e8_0.605.pth\r\n",
      " task1_BERTW_merged_all_data_freeze_3_e8_0.61.pth\r\n",
      " task1_BERTW_merged_all_data+PTC_freeze_3_e6_0.611.pth\r\n",
      " task1_BERTW_merged_all_data_warmup2_e8_0.614.pth\r\n",
      " task1_BERTW_merged_data_BCE_weighted_e9_0.418.pth\r\n",
      "'task1_BERTW_merged_data_complex_loss*0.1_e6_0.59.pth'\r\n",
      "'task1_BERTW_merged_data_complex_loss*0.5_e9_0.612.pth'\r\n",
      "'task1_BERTW_merged_data_complex_loss*5_e9_0.596.pth'\r\n",
      " task1_BERTW_merged_data_complex_loss_e3_0.585.pth\r\n",
      " task1_BERTW_merged_data_complex_loss_e7_0.606.pth\r\n",
      " task1_BERTW_merged_data_dropout_e7_0.61.pth\r\n",
      " task1_BERTW_merged_data_freeze_10_e8_0.595.pth\r\n",
      " task1_BERTW_merged_data_freeze_3_e9_0.604.pth\r\n",
      " task1_BERTW_merged_data_freeze_3_warmup_e13_0.611.pth\r\n",
      " task1_BERTW_merged_data_freeze_6_e9_0.613.pth\r\n",
      "'task1_BERTW_merged_data+PTC_complex_loss*0.5_e9_0.612.pth'\r\n",
      " task1_BERTW_merged_data_seed42_e9_0.619.pth\r\n",
      " task1_BERTW_onyly_prop_weights_merged_all_data_e9_0.596.pth\r\n",
      " task1_BERTWPTC_merged_all_data_e8_0.588.pth\r\n",
      " task1_DebertaW_merged_all_data_e2_0.625.pth\r\n",
      " task1_DistilBertW_merged_all_data_e8_0.607.pth\r\n",
      " task1_HateMemeW_merged_all_data_e6_0.613.pth\r\n",
      " task1_ROBERTAW_e21_0.519.pth\r\n",
      " task1_RoBERTaW_merged_all_data_e5_0.628.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls /data/peskine/semeval2024/models/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "4c9be74e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bert(\n",
       "  (bert): BertForSequenceClassification(\n",
       "    (bert): BertModel(\n",
       "      (embeddings): BertEmbeddings(\n",
       "        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "        (position_embeddings): Embedding(512, 768)\n",
       "        (token_type_embeddings): Embedding(2, 768)\n",
       "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (encoder): BertEncoder(\n",
       "        (layer): ModuleList(\n",
       "          (0): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (1): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (2): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (3): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (4): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (5): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (6): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (7): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (8): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (9): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (10): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (11): BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (pooler): BertPooler(\n",
       "        (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (activation): Tanh()\n",
       "      )\n",
       "    )\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "    (classifier): Linear(in_features=768, out_features=56, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load(path+'best_model.pth'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "1dd4a9c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t Eval loss: 28.09398991721017\n",
      "\t Eval F1H: 0.597, PREH: 0.574, RECH: 0.622\n",
      "\t Eval F1s: [0.283, 0.0, 0.428, 0.273, 0.822, 0.437, 0.195, 0.196, 0.352, 0.197, 0.105, 0.0, 0.273, 0.389, 0.241, 0.55, 0.532, 0.2, 0.39, 0.604, 0.708, 0.514, 0.666, 0.455, 0.279, 0.774, 0.685, 0.671]\n",
      "---------------------------------------------------------------------------\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "\n",
    "predictions_sep = [[], [], [], [], [], [], [], [], [],[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]\n",
    "\n",
    "labels_sep = [[], [], [], [], [], [], [], [], [],[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]\n",
    "\n",
    "eval_loss = 0\n",
    "steps=0\n",
    "for step, batch in enumerate(dev_dataloader):\n",
    "\n",
    "    # Add batch to GPU\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "\n",
    "    b_input_ids, b_input_mask, b_labels, b_token_type_ids = batch\n",
    "\n",
    "    b_labels = b_labels.float()\n",
    "\n",
    "    with torch.no_grad():\n",
    "\n",
    "        logits = model(b_input_ids, b_token_type_ids, b_input_mask)\n",
    "        #logits = model(b_input_ids, b_token_type_ids, b_input_mask).logits\n",
    "\n",
    "\n",
    "\n",
    "    logits = logits.detach().cpu().numpy()\n",
    "    ground_truth = b_labels.detach().cpu().numpy()\n",
    "\n",
    "    steps+=1\n",
    "\n",
    "    for i in range(0, len(TECHNIQUES)):\n",
    "        for p in logits:\n",
    "            p_i = p[2*i:2*i+2]\n",
    "            pred = np.argmax(p_i)\n",
    "            predictions_sep[i].append(pred)\n",
    "        #for p in logits:\n",
    "        #    p_sig = sig(torch.Tensor(p))\n",
    "        #    p_01 = (p_sig>0.5).int()\n",
    "        #    predictions_sep[i].append(p_01[i].item())\n",
    "\n",
    "        for l in ground_truth:\n",
    "            labels_sep[i].append(l[i])\n",
    "\n",
    "\n",
    "precision, recall, f1_h = evaluate_preds(predictions_sep, dev)\n",
    "F1s = []\n",
    "for i in range(0, len(TECHNIQUES)):\n",
    "    F1s.append(round(metrics.f1_score(labels_sep[i], predictions_sep[i]), 3))\n",
    "\n",
    "\n",
    "print(\"\\t Eval loss: {}\".format(LOSS))\n",
    "print(\"\\t Eval F1H: {}, PREH: {}, RECH: {}\".format(round(f1_h, 3), round(precision, 3), round(recall, 3)))\n",
    "print(\"\\t Eval F1s: {}\".format(F1s))\n",
    "print(\"---\"*25)\n",
    "print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "bbbcf515",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.597%0.574%0.622%0.283%0.0%0.428%0.273%0.822%0.437%0.195%0.196%0.352%0.197%0.105%0.0%0.273%0.389%0.241%0.55%0.532%0.2%0.39%0.604%0.708%0.514%0.666%0.455%0.279%0.774%0.685%0.671'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "METRICS = [f1_h, precision, recall]\n",
    "METRICS.extend(F1s)\n",
    "METRICS_STR = []\n",
    "for m in METRICS:\n",
    "    METRICS_STR.append(str(round(m, 3)))\n",
    "\n",
    "\"%\".join(METRICS_STR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d8a1ecdd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Repetition : 0.463\n",
      "Obfuscation, Intentional vagueness, Confusion : 0.0\n",
      "Slogans : 0.462\n",
      "Bandwagon : 0.286\n",
      "Appeal to authority : 0.835\n",
      "Flag-waving : 0.503\n",
      "Appeal to fear/prejudice : 0.341\n",
      "Causal Oversimplification : 0.101\n",
      "Black-and-white Fallacy/Dictatorship : 0.422\n",
      "Thought-terminating cliché : 0.306\n",
      "Misrepresentation of Someone's Position (Straw Man) : 0.111\n",
      "Presenting Irrelevant Data (Red Herring) : 0.0\n",
      "Whataboutism : 0.326\n",
      "Glittering generalities (Virtue) : 0.541\n",
      "Doubt : 0.355\n",
      "Name calling/Labeling : 0.575\n",
      "Smears : 0.535\n",
      "Reductio ad hitlerum : 0.133\n",
      "Exaggeration/Minimisation : 0.423\n",
      "Loaded Language : 0.627\n"
     ]
    }
   ],
   "source": [
    "for i in range(0, len(TECHNIQUES)):\n",
    "    print(TECHNIQUES[i], ':', F1s[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bd244c3",
   "metadata": {},
   "source": [
    "# Generate submission file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "285221f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "PREDS_NAMES = convert_numbers_to_name(predictions_sep)\n",
    "submission = []\n",
    "for i in range(0, len(dev)):\n",
    "    d = {}\n",
    "    d['id'] = dev[i]['id']\n",
    "    d['labels'] = PREDS_NAMES[i]\n",
    "    submission.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce8d2338",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path+'submission_test_v2.txt', 'w') as f:\n",
    "    json.dump(submission, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}