demo_fastbert_sample.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "demo-fastbert-sample.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/wshuyi/demo-fastbert-multi-label-classification/blob/master/demo_fastbert_sample.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3LmH5JiJ6SJy",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title\n",
        "!git clone https://github.com/NVIDIA/apex.git\n",
        "%cd apex\n",
        "!pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .\n",
        "%cd ..\n",
        "\n",
        "import torch\n",
        "import apex\n",
        "from fastai.text import *\n",
        "import datetime\n",
        "run_start_time = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M-%S')\n",
        "\n",
        "LOG_PATH=Path('logs/')  \n",
        "MODEL_PATH=Path('models/') \n",
        "\n",
        "if not LOG_PATH.exists():\n",
        "  LOG_PATH.mkdir()\n",
        "import logging\n",
        "\n",
        "args = {\n",
        "    \"run_text\": \"my_test\",\n",
        "    \"max_seq_length\": 512,\n",
        "    \"do_lower_case\": True,\n",
        "    \"train_batch_size\": 16,\n",
        "    \"learning_rate\": 6e-5,\n",
        "    \"num_train_epochs\": 12.0,\n",
        "    \"warmup_proportion\": 0.002,\n",
        "    \"local_rank\": -1,\n",
        "    \"gradient_accumulation_steps\": 1,\n",
        "    \"fp16\": True,\n",
        "    \"loss_scale\": 128\n",
        "}\n",
        "\n",
        "logfile = str(LOG_PATH/'log-{}-{}.txt'.format(run_start_time, args[\"run_text\"]))\n",
        "\n",
        "logging.basicConfig(\n",
        "    level=logging.INFO,\n",
        "    format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n",
        "    datefmt='%m/%d/%Y %H:%M:%S',\n",
        "    handlers=[\n",
        "        logging.FileHandler(logfile),\n",
        "        logging.StreamHandler(sys.stdout)\n",
        "    ])\n",
        "\n",
        "logger = logging.getLogger()\n",
        "\n",
        "device = torch.device('cuda')\n",
        "\n",
        "if torch.cuda.device_count() > 1:\n",
        "    multi_gpu = True\n",
        "else:\n",
        "    multi_gpu = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9cWcRyZDuc11",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/wshuyi/demo-multi-label-classification-bert.git"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4q4BrLHFzvW-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install fast-bert"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7AJoU_jWn_Oi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from fast_bert.data import *\n",
        "from fast_bert.learner import *\n",
        "from fast_bert.metrics import *\n",
        "from pytorch_pretrained_bert.tokenization import BertTokenizer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6en9qPiQogbT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "DATA_PATH = Path('demo-multi-label-classification-bert/sample/data/')     \n",
        "LABEL_PATH = Path('demo-multi-label-classification-bert/sample/labels/')  \n",
        "\n",
        "BERT_PRETRAINED_MODEL = \"bert-base-uncased\"\n",
        "\n",
        "args[\"do_lower_case\"] = True\n",
        "args[\"train_batch_size\"] = 16\n",
        "args[\"learning_rate\"] = 6e-5\n",
        "args[\"max_seq_length\"] = 512\n",
        "args[\"fp16\"] = True"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QgG5OHRMo98N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained(BERT_PRETRAINED_MODEL, \n",
        "                                          do_lower_case=args['do_lower_case'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VM-gQVUfwmns",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "label_cols = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1uXL6R-4vvnF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "databunch = BertDataBunch(DATA_PATH, LABEL_PATH, tokenizer, train_file='train.csv', val_file='valid.csv',\n",
        "                          test_data='test.csv', label_file=\"labels.csv\",\n",
        "                          text_col=\"comment_text\", label_col=label_cols,\n",
        "                          bs=args['train_batch_size'], maxlen=args['max_seq_length'], \n",
        "                          multi_gpu=multi_gpu, multi_label=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YUKO6P4Twf50",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "metrics = [{'name': 'accuracy', 'function': accuracy_multilabel}]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OrbgeJvs0uRT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "logger.info(args)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KVhTUgSf4WJB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learner = BertLearner.from_pretrained_model(databunch, BERT_PRETRAINED_MODEL, metrics, device, logger, \n",
        "                                            is_fp16=args['fp16'], loss_scale=args['loss_scale'], \n",
        "                                            multi_gpu=multi_gpu,  multi_label=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U5hl8TZt4ofa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learner.fit(4, lr=args['learning_rate'], schedule_type=\"warmup_linear\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tWckPi8O6fxW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}