main.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# W4_EIG_CUDA\n",
        "\n",
        "# !python opt.py facebook/opt-125m c4 --wbits 4 --nsamples 128 --eigenvalues 1 --save opt-125m-w4-eig-n128.pt\n",
        "# !python opt.py facebook/opt-125m c4 --load opt-125m-w4-eig-n128.pt --benchmark 128 --check\n"
      ],
      "metadata": {
        "collapsed": true,
        "id": "Jinz1jG-AjAT",
        "outputId": "6119959a-fe81-44ac-c30b-14c5e9dc06d9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Loading model ...\n",
            "Done.\n",
            "Processed 256 sequences.\n",
            "Benchmarking ...\n",
            "0 0.831397294998169\n",
            "1 0.017699480056762695\n",
            "2 0.011807441711425781\n",
            "3 0.011462688446044922\n",
            "4 0.01158905029296875\n",
            "5 0.011292695999145508\n",
            "6 0.011590242385864258\n",
            "7 0.011566638946533203\n",
            "8 0.011832952499389648\n",
            "9 0.011565446853637695\n",
            "10 0.011381864547729492\n",
            "11 0.014292716979980469\n",
            "12 0.011407613754272461\n",
            "13 0.012975454330444336\n",
            "14 0.0117340087890625\n",
            "15 0.01159977912902832\n",
            "16 0.011457204818725586\n",
            "17 0.014764785766601562\n",
            "18 0.011444568634033203\n",
            "19 0.011433124542236328\n",
            "20 0.011245965957641602\n",
            "21 0.011084794998168945\n",
            "22 0.011439323425292969\n",
            "23 0.011127948760986328\n",
            "24 0.011061906814575195\n",
            "25 0.011835098266601562\n",
            "26 0.011204957962036133\n",
            "27 0.01095438003540039\n",
            "28 0.011092424392700195\n",
            "29 0.011176109313964844\n",
            "30 0.011512279510498047\n",
            "31 0.011035442352294922\n",
            "32 0.01116943359375\n",
            "33 0.011095762252807617\n",
            "34 0.011244773864746094\n",
            "35 0.012166976928710938\n",
            "36 0.010875463485717773\n",
            "37 0.010877132415771484\n",
            "38 0.011971473693847656\n",
            "39 0.011122465133666992\n",
            "40 0.011235713958740234\n",
            "41 0.01093912124633789\n",
            "42 0.011129140853881836\n",
            "43 0.015032291412353516\n",
            "44 0.01173710823059082\n",
            "45 0.011332988739013672\n",
            "46 0.011768579483032227\n",
            "47 0.011703968048095703\n",
            "48 0.011925458908081055\n",
            "49 0.011599302291870117\n",
            "50 0.011648178100585938\n",
            "51 0.012614011764526367\n",
            "52 0.011849641799926758\n",
            "53 0.01204681396484375\n",
            "54 0.011926412582397461\n",
            "55 0.01322627067565918\n",
            "56 0.011496543884277344\n",
            "57 0.011512041091918945\n",
            "58 0.01426839828491211\n",
            "59 0.012881040573120117\n",
            "60 0.011568546295166016\n",
            "61 0.012466669082641602\n",
            "62 0.011358022689819336\n",
            "63 0.011179924011230469\n",
            "64 0.011321544647216797\n",
            "65 0.014832258224487305\n",
            "66 0.016893386840820312\n",
            "67 0.013945817947387695\n",
            "68 0.01112222671508789\n",
            "69 0.011227607727050781\n",
            "70 0.011090993881225586\n",
            "71 0.011141538619995117\n",
            "72 0.011354684829711914\n",
            "73 0.011265754699707031\n",
            "74 0.01127004623413086\n",
            "75 0.01170039176940918\n",
            "76 0.011275529861450195\n",
            "77 0.0113525390625\n",
            "78 0.011456727981567383\n",
            "79 0.011574983596801758\n",
            "80 0.011186838150024414\n",
            "81 0.01132965087890625\n",
            "82 0.011995077133178711\n",
            "83 0.011333227157592773\n",
            "84 0.01176595687866211\n",
            "85 0.011271953582763672\n",
            "86 0.011050224304199219\n",
            "87 0.011002063751220703\n",
            "88 0.010961294174194336\n",
            "89 0.01085209846496582\n",
            "90 0.01119089126586914\n",
            "91 0.011097431182861328\n",
            "92 0.013153314590454102\n",
            "93 0.016063451766967773\n",
            "94 0.014877080917358398\n",
            "95 0.012727499008178711\n",
            "96 0.013901233673095703\n",
            "97 0.011605501174926758\n",
            "98 0.011916160583496094\n",
            "99 0.011325597763061523\n",
            "100 0.014724969863891602\n",
            "101 0.011389493942260742\n",
            "102 0.011327505111694336\n",
            "103 0.011171817779541016\n",
            "104 0.011240482330322266\n",
            "105 0.011387348175048828\n",
            "106 0.011356115341186523\n",
            "107 0.011565923690795898\n",
            "108 0.011318206787109375\n",
            "109 0.012551546096801758\n",
            "110 0.011337757110595703\n",
            "111 0.011017084121704102\n",
            "112 0.011254072189331055\n",
            "113 0.012021780014038086\n",
            "114 0.011780500411987305\n",
            "115 0.011448144912719727\n",
            "116 0.011240243911743164\n",
            "117 0.014292001724243164\n",
            "118 0.011391401290893555\n",
            "119 0.011313915252685547\n",
            "120 0.011270761489868164\n",
            "121 0.011307716369628906\n",
            "122 0.011327743530273438\n",
            "123 0.011037349700927734\n",
            "124 0.011013507843017578\n",
            "125 0.011064291000366211\n",
            "126 0.011645317077636719\n",
            "127 0.012410879135131836\n",
            "Median: 0.011420369148254395\n",
            "PPL: 16125.560546875\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# SEPARATE PARTS"
      ],
      "metadata": {
        "id": "eRlxAkKBm55O"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## W4\n"
      ],
      "metadata": {
        "id": "AF_RqVPNpBl9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!python opt.py facebook/opt-125m c4 --wbits 4 --nsamples 128 --save opt-125m-w4-n128.pt\n"
      ],
      "metadata": {
        "collapsed": true,
        "id": "x6aCjpyVo5vY",
        "outputId": "04938b69-a757-4e00-ef07-34c0c1a6407e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processed 256 sequences.\n",
            "Starting ...\n",
            "Ready.\n",
            "iterating decoder layers...\n",
            "USING OBS ACTIVATION Q\n",
            "quantizing weights\n",
            "[1 / 12] |  time 0.52 | error 17947.017578125 | self_attn.k_proj\n",
            "[1 / 12] |  time 0.31 | error 186.532470703125 | self_attn.v_proj\n",
            "[1 / 12] |  time 0.31 | error 12722.03125 | self_attn.q_proj\n",
            "[1 / 12] |  time 0.31 | error 4.513874530792236 | self_attn.out_proj\n",
            "[1 / 12] |  time 0.38 | error 517.441162109375 | fc1\n",
            "[1 / 12] |  time 1.33 | error 28.09429168701172 | fc2\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/gptaq/opt.py\", line 564, in <module>\n",
            "    quantizers = opt_sequential(model, dataloader, DEV)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py\", line 115, in decorate_context\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/content/gptaq/opt.py\", line 156, in opt_sequential\n",
            "    outs[j] = layer(inps[j].unsqueeze(0), attention_mask=attention_mask)[0]\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/models/opt/modeling_opt.py\", line 524, in forward\n",
            "    hidden_states, self_attn_weights, present_key_value = self.self_attn(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/models/opt/modeling_opt.py\", line 154, in forward\n",
            "    query_states = self.q_proj(hidden_states) * self.scaling\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1595, in _call_impl\n",
            "    hook_result = hook(self, args, result)\n",
            "  File \"/content/gptaq/opt.py\", line 135, in tmp\n",
            "    gptaq[name].after_forward(inp[0].data, out.data)\n",
            "  File \"/content/gptaq/gptaq.py\", line 62, in after_forward\n",
            "    return super().add_batch(inp, out)\n",
            "  File \"/content/gptaq/gptq.py\", line 54, in add_batch\n",
            "    self.H *= self.nsamples / (self.nsamples + tmp)\n",
            "AttributeError: 'GPTAQ' object has no attribute 'H'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python opt.py facebook/opt-125m c4 --load opt-125m-w4-n128.pt --benchmark 128 --check\n"
      ],
      "metadata": {
        "id": "EKfiRi-4pEDd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## W4_A4_RTN\n"
      ],
      "metadata": {
        "id": "J0Ga0xeZpCZO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!python opt.py facebook/opt-125m c4 --wbits 4 --nsamples 128 --abits 4 --amethod rtn --save opt-125m-w4-a4-rtn-n128.pt\n"
      ],
      "metadata": {
        "id": "gZJUrgI9o8QE",
        "outputId": "ceed8d62-fdd0-4e82-ced6-4189b32adede",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processed 256 sequences.\n",
            "Starting ...\n",
            "Ready.\n",
            "iterating decoder layers...\n",
            "USING RTN ACTIVATION Q\n",
            "quantizing weights\n",
            "[1 / 12] |  time 0.48 | error 11456.970703125 | self_attn.k_proj\n",
            "[1 / 12] |  time 0.30 | error 97.91262817382812 | self_attn.v_proj\n",
            "[1 / 12] |  time 0.32 | error 8083.791015625 | self_attn.q_proj\n",
            "[1 / 12] |  time 0.31 | error 2.710146427154541 | self_attn.out_proj\n",
            "[1 / 12] |  time 0.39 | error 242.84774780273438 | fc1\n",
            "[1 / 12] |  time 1.26 | error 10.808691024780273 | fc2\n",
            "quantizing activations ...\n",
            "[2 / 12] |  time 0.41 | error 6765.34716796875 | self_attn.k_proj\n",
            "[2 / 12] |  time 0.40 | error 84.13201141357422 | self_attn.v_proj\n",
            "[2 / 12] |  time 0.46 | error 3603.398193359375 | self_attn.q_proj\n",
            "[2 / 12] |  time 0.47 | error 2.763556480407715 | self_attn.out_proj\n",
            "[2 / 12] |  time 0.53 | error 1144.73388671875 | fc1\n",
            "[2 / 12] |  time 1.17 | error 9.255569458007812 | fc2\n",
            "quantizing activations ...\n",
            "[3 / 12] |  time 0.30 | error 12509.92578125 | self_attn.k_proj\n",
            "[3 / 12] |  time 0.30 | error 245.31610107421875 | self_attn.v_proj\n",
            "[3 / 12] |  time 0.30 | error 8946.1865234375 | self_attn.q_proj\n",
            "[3 / 12] |  time 0.31 | error 1.910596251487732 | self_attn.out_proj\n",
            "[3 / 12] |  time 0.39 | error 1029.4010009765625 | fc1\n",
            "[3 / 12] |  time 1.19 | error 13.768155097961426 | fc2\n",
            "quantizing activations ...\n",
            "[4 / 12] |  time 0.45 | error 17389.52734375 | self_attn.k_proj\n",
            "[4 / 12] |  time 0.46 | error 319.5351257324219 | self_attn.v_proj\n",
            "[4 / 12] |  time 0.30 | error 14941.64453125 | self_attn.q_proj\n",
            "[4 / 12] |  time 0.31 | error 2.0757546424865723 | self_attn.out_proj\n",
            "[4 / 12] |  time 0.37 | error 670.9038696289062 | fc1\n",
            "[4 / 12] |  time 1.20 | error 7.102419376373291 | fc2\n",
            "quantizing activations ...\n",
            "[5 / 12] |  time 0.31 | error 18800.099609375 | self_attn.k_proj\n",
            "[5 / 12] |  time 0.32 | error 391.837890625 | self_attn.v_proj\n",
            "[5 / 12] |  time 0.31 | error 14433.0234375 | self_attn.q_proj\n",
            "[5 / 12] |  time 0.30 | error 2.7953455448150635 | self_attn.out_proj\n",
            "[5 / 12] |  time 0.38 | error 1041.7071533203125 | fc1\n",
            "[5 / 12] |  time 1.17 | error 13.707175254821777 | fc2\n",
            "quantizing activations ...\n",
            "[6 / 12] |  time 0.32 | error 18869.923828125 | self_attn.k_proj\n",
            "[6 / 12] |  time 0.30 | error 335.83056640625 | self_attn.v_proj\n",
            "[6 / 12] |  time 0.32 | error 15196.3994140625 | self_attn.q_proj\n",
            "[6 / 12] |  time 0.32 | error 4.56256103515625 | self_attn.out_proj\n",
            "[6 / 12] |  time 0.35 | error 811.8319091796875 | fc1\n",
            "[6 / 12] |  time 1.20 | error 20.433406829833984 | fc2\n",
            "quantizing activations ...\n",
            "[7 / 12] |  time 0.31 | error 20162.578125 | self_attn.k_proj\n",
            "[7 / 12] |  time 0.30 | error 447.8818359375 | self_attn.v_proj\n",
            "[7 / 12] |  time 0.31 | error 13925.78515625 | self_attn.q_proj\n",
            "[7 / 12] |  time 0.31 | error 5.549628257751465 | self_attn.out_proj\n",
            "[7 / 12] |  time 0.36 | error 814.456787109375 | fc1\n",
            "[7 / 12] |  time 1.20 | error 26.34343719482422 | fc2\n",
            "quantizing activations ...\n",
            "[8 / 12] |  time 0.31 | error 17491.384765625 | self_attn.k_proj\n",
            "[8 / 12] |  time 0.30 | error 542.3739013671875 | self_attn.v_proj\n",
            "[8 / 12] |  time 0.30 | error 12267.275390625 | self_attn.q_proj\n",
            "[8 / 12] |  time 0.30 | error 11.14691162109375 | self_attn.out_proj\n",
            "[8 / 12] |  time 0.36 | error 1003.5086059570312 | fc1\n",
            "[8 / 12] |  time 1.17 | error 26.98171615600586 | fc2\n",
            "quantizing activations ...\n",
            "[9 / 12] |  time 0.31 | error 19111.34375 | self_attn.k_proj\n",
            "[9 / 12] |  time 0.30 | error 710.640869140625 | self_attn.v_proj\n",
            "[9 / 12] |  time 0.31 | error 12993.373046875 | self_attn.q_proj\n",
            "[9 / 12] |  time 0.31 | error 19.16356658935547 | self_attn.out_proj\n",
            "[9 / 12] |  time 0.36 | error 1305.7120361328125 | fc1\n",
            "[9 / 12] |  time 1.38 | error 49.10004806518555 | fc2\n",
            "quantizing activations ...\n",
            "[10 / 12] |  time 0.33 | error 15609.498046875 | self_attn.k_proj\n",
            "[10 / 12] |  time 0.35 | error 757.79541015625 | self_attn.v_proj\n",
            "[10 / 12] |  time 0.33 | error 9367.478515625 | self_attn.q_proj\n",
            "[10 / 12] |  time 0.30 | error 34.18034362792969 | self_attn.out_proj\n",
            "[10 / 12] |  time 0.36 | error 1641.237548828125 | fc1\n",
            "[10 / 12] |  time 1.18 | error 56.022098541259766 | fc2\n",
            "quantizing activations ...\n",
            "[11 / 12] |  time 0.30 | error 16953.888671875 | self_attn.k_proj\n",
            "[11 / 12] |  time 0.30 | error 918.8005981445312 | self_attn.v_proj\n",
            "[11 / 12] |  time 0.31 | error 10852.232421875 | self_attn.q_proj\n",
            "[11 / 12] |  time 0.45 | error 19.12511444091797 | self_attn.out_proj\n",
            "[11 / 12] |  time 0.53 | error 1909.841064453125 | fc1\n",
            "[11 / 12] |  time 1.76 | error 104.78924560546875 | fc2\n",
            "quantizing activations ...\n",
            "[12 / 12] |  time 0.30 | error 9039.642578125 | self_attn.k_proj\n",
            "[12 / 12] |  time 0.32 | error 1154.8785400390625 | self_attn.v_proj\n",
            "[12 / 12] |  time 0.32 | error 6119.2421875 | self_attn.q_proj\n",
            "[12 / 12] |  time 0.32 | error 32.997962951660156 | self_attn.out_proj\n",
            "[12 / 12] |  time 0.37 | error 2724.12939453125 | fc1\n",
            "[12 / 12] |  time 1.19 | error 171.06704711914062 | fc2\n",
            "quantizing activations ...\n",
            "84 s\n",
            "/usr/local/lib/python3.10/dist-packages/datasets/load.py:1491: FutureWarning: The repository for ptb_text_only contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/ptb_text_only\n",
            "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
            "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
            "  warnings.warn(\n",
            "ptb\n",
            "Evaluating ...\n",
            "[0 / 12]\n",
            "[1 / 12]\n",
            "[2 / 12]\n",
            "[3 / 12]\n",
            "[4 / 12]\n",
            "[5 / 12]\n",
            "[6 / 12]\n",
            "[7 / 12]\n",
            "[8 / 12]\n",
            "[9 / 12]\n",
            "[10 / 12]\n",
            "[11 / 12]\n",
            "52 PPL\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Packing ...\n",
            "model.decoder.layers.0.self_attn.k_proj\n",
            "model.decoder.layers.0.self_attn.v_proj\n",
            "model.decoder.layers.0.self_attn.q_proj\n",
            "model.decoder.layers.0.self_attn.out_proj\n",
            "model.decoder.layers.0.fc1\n",
            "model.decoder.layers.0.fc2\n",
            "model.decoder.layers.1.self_attn.k_proj\n",
            "model.decoder.layers.1.self_attn.v_proj\n",
            "model.decoder.layers.1.self_attn.q_proj\n",
            "model.decoder.layers.1.self_attn.out_proj\n",
            "model.decoder.layers.1.fc1\n",
            "model.decoder.layers.1.fc2\n",
            "model.decoder.layers.2.self_attn.k_proj\n",
            "model.decoder.layers.2.self_attn.v_proj\n",
            "model.decoder.layers.2.self_attn.q_proj\n",
            "model.decoder.layers.2.self_attn.out_proj\n",
            "model.decoder.layers.2.fc1\n",
            "model.decoder.layers.2.fc2\n",
            "model.decoder.layers.3.self_attn.k_proj\n",
            "model.decoder.layers.3.self_attn.v_proj\n",
            "model.decoder.layers.3.self_attn.q_proj\n",
            "model.decoder.layers.3.self_attn.out_proj\n",
            "model.decoder.layers.3.fc1\n",
            "model.decoder.layers.3.fc2\n",
            "model.decoder.layers.4.self_attn.k_proj\n",
            "model.decoder.layers.4.self_attn.v_proj\n",
            "model.decoder.layers.4.self_attn.q_proj\n",
            "model.decoder.layers.4.self_attn.out_proj\n",
            "model.decoder.layers.4.fc1\n",
            "model.decoder.layers.4.fc2\n",
            "model.decoder.layers.5.self_attn.k_proj\n",
            "model.decoder.layers.5.self_attn.v_proj\n",
            "model.decoder.layers.5.self_attn.q_proj\n",
            "model.decoder.layers.5.self_attn.out_proj\n",
            "model.decoder.layers.5.fc1\n",
            "model.decoder.layers.5.fc2\n",
            "model.decoder.layers.6.self_attn.k_proj\n",
            "model.decoder.layers.6.self_attn.v_proj\n",
            "model.decoder.layers.6.self_attn.q_proj\n",
            "model.decoder.layers.6.self_attn.out_proj\n",
            "model.decoder.layers.6.fc1\n",
            "model.decoder.layers.6.fc2\n",
            "model.decoder.layers.7.self_attn.k_proj\n",
            "model.decoder.layers.7.self_attn.v_proj\n",
            "model.decoder.layers.7.self_attn.q_proj\n",
            "model.decoder.layers.7.self_attn.out_proj\n",
            "model.decoder.layers.7.fc1\n",
            "model.decoder.layers.7.fc2\n",
            "model.decoder.layers.8.self_attn.k_proj\n",
            "model.decoder.layers.8.self_attn.v_proj\n",
            "model.decoder.layers.8.self_attn.q_proj\n",
            "model.decoder.layers.8.self_attn.out_proj\n",
            "model.decoder.layers.8.fc1\n",
            "model.decoder.layers.8.fc2\n",
            "model.decoder.layers.9.self_attn.k_proj\n",
            "model.decoder.layers.9.self_attn.v_proj\n",
            "model.decoder.layers.9.self_attn.q_proj\n",
            "model.decoder.layers.9.self_attn.out_proj\n",
            "model.decoder.layers.9.fc1\n",
            "model.decoder.layers.9.fc2\n",
            "model.decoder.layers.10.self_attn.k_proj\n",
            "model.decoder.layers.10.self_attn.v_proj\n",
            "model.decoder.layers.10.self_attn.q_proj\n",
            "model.decoder.layers.10.self_attn.out_proj\n",
            "model.decoder.layers.10.fc1\n",
            "model.decoder.layers.10.fc2\n",
            "model.decoder.layers.11.self_attn.k_proj\n",
            "model.decoder.layers.11.self_attn.v_proj\n",
            "model.decoder.layers.11.self_attn.q_proj\n",
            "model.decoder.layers.11.self_attn.out_proj\n",
            "model.decoder.layers.11.fc1\n",
            "model.decoder.layers.11.fc2\n",
            "Done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python opt.py facebook/opt-125m c4 --load opt-125m-w4-a4-rtn-n128.pt --benchmark 128 --check\n"
      ],
      "metadata": {
        "id": "_hRnq5NipGrB",
        "outputId": "1b32a096-0bc1-462f-b499-4622817cd5f0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Loading model ...\n",
            "Done.\n",
            "Processed 256 sequences.\n",
            "Benchmarking ...\n",
            "0 0.6263577938079834\n",
            "1 0.01600360870361328\n",
            "2 0.011171817779541016\n",
            "3 0.010941743850708008\n",
            "4 0.012329578399658203\n",
            "5 0.011235475540161133\n",
            "6 0.011223077774047852\n",
            "7 0.01126718521118164\n",
            "8 0.017355918884277344\n",
            "9 0.015521049499511719\n",
            "10 0.011439800262451172\n",
            "11 0.011363983154296875\n",
            "12 0.011690616607666016\n",
            "13 0.01150655746459961\n",
            "14 0.011306524276733398\n",
            "15 0.011406898498535156\n",
            "16 0.011528491973876953\n",
            "17 0.01175379753112793\n",
            "18 0.011386632919311523\n",
            "19 0.011413812637329102\n",
            "20 0.011234283447265625\n",
            "21 0.015059947967529297\n",
            "22 0.011094331741333008\n",
            "23 0.012053489685058594\n",
            "24 0.011260032653808594\n",
            "25 0.011879682540893555\n",
            "26 0.011486530303955078\n",
            "27 0.011345624923706055\n",
            "28 0.011520147323608398\n",
            "29 0.01279449462890625\n",
            "30 0.011852502822875977\n",
            "31 0.011271238327026367\n",
            "32 0.012582063674926758\n",
            "33 0.011364221572875977\n",
            "34 0.0117645263671875\n",
            "35 0.011859893798828125\n",
            "36 0.011498689651489258\n",
            "37 0.01244974136352539\n",
            "38 0.011420965194702148\n",
            "39 0.01127934455871582\n",
            "40 0.011056900024414062\n",
            "41 0.011059045791625977\n",
            "42 0.011233329772949219\n",
            "43 0.01143503189086914\n",
            "44 0.011349678039550781\n",
            "45 0.011815309524536133\n",
            "46 0.017345190048217773\n",
            "47 0.011693000793457031\n",
            "48 0.012270212173461914\n",
            "49 0.012188196182250977\n",
            "50 0.012827634811401367\n",
            "51 0.012674808502197266\n",
            "52 0.012723684310913086\n",
            "53 0.011889934539794922\n",
            "54 0.011385679244995117\n",
            "55 0.011247873306274414\n",
            "56 0.013216972351074219\n",
            "57 0.011580467224121094\n",
            "58 0.012612581253051758\n",
            "59 0.011557817459106445\n",
            "60 0.011365890502929688\n",
            "61 0.011893749237060547\n",
            "62 0.012119054794311523\n",
            "63 0.013510704040527344\n",
            "64 0.011333465576171875\n",
            "65 0.013003826141357422\n",
            "66 0.011200189590454102\n",
            "67 0.011084318161010742\n",
            "68 0.011145591735839844\n",
            "69 0.011432170867919922\n",
            "70 0.011953592300415039\n",
            "71 0.014847755432128906\n",
            "72 0.011660575866699219\n",
            "73 0.011292219161987305\n",
            "74 0.011606454849243164\n",
            "75 0.011178970336914062\n",
            "76 0.011133909225463867\n",
            "77 0.011637210845947266\n",
            "78 0.0114898681640625\n",
            "79 0.011412382125854492\n",
            "80 0.011639833450317383\n",
            "81 0.010946035385131836\n",
            "82 0.011204719543457031\n",
            "83 0.011144161224365234\n",
            "84 0.011501789093017578\n",
            "85 0.011467933654785156\n",
            "86 0.011600732803344727\n",
            "87 0.011209249496459961\n",
            "88 0.01429295539855957\n",
            "89 0.011560440063476562\n",
            "90 0.01161050796508789\n",
            "91 0.011370420455932617\n",
            "92 0.01372075080871582\n",
            "93 0.018270492553710938\n",
            "94 0.011433839797973633\n",
            "95 0.011338472366333008\n",
            "96 0.015122652053833008\n",
            "97 0.011409997940063477\n",
            "98 0.011806011199951172\n",
            "99 0.011706352233886719\n",
            "100 0.011523246765136719\n",
            "101 0.011308670043945312\n",
            "102 0.012128114700317383\n",
            "103 0.011707782745361328\n",
            "104 0.011998414993286133\n",
            "105 0.012508630752563477\n",
            "106 0.011329412460327148\n",
            "107 0.011219501495361328\n",
            "108 0.01102304458618164\n",
            "109 0.011181116104125977\n",
            "110 0.011120796203613281\n",
            "111 0.011077880859375\n",
            "112 0.011310815811157227\n",
            "113 0.014284372329711914\n",
            "114 0.011753320693969727\n",
            "115 0.01207280158996582\n",
            "116 0.012067794799804688\n",
            "117 0.011585474014282227\n",
            "118 0.014428853988647461\n",
            "119 0.013683557510375977\n",
            "120 0.015288591384887695\n",
            "121 0.011289358139038086\n",
            "122 0.011182069778442383\n",
            "123 0.011463642120361328\n",
            "124 0.011113166809082031\n",
            "125 0.011799335479736328\n",
            "126 0.011538267135620117\n",
            "127 0.012167215347290039\n",
            "Median: 0.011525869369506836\n",
            "PPL: 14716.4013671875\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## W4_A4_TOKEN\n"
      ],
      "metadata": {
        "id": "fKOtvVHQpDL0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!python opt.py facebook/opt-125m c4 --wbits 4 --nsamples 128 --abits 4 --amethod token --save opt-125m-w4-a4-token-n128.pt\n"
      ],
      "metadata": {
        "id": "widUooXyo9HB",
        "outputId": "df38a65d-7e32-46de-bbab-c8fb68b1e1b2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processed 256 sequences.\n",
            "Starting ...\n",
            "Ready.\n",
            "iterating decoder layers...\n",
            "USING TOKEN-WISE ACTIVATION Q\n",
            "quantizing weights\n",
            "[1 / 12] |  time 0.48 | error 11456.970703125 | self_attn.k_proj\n",
            "[1 / 12] |  time 0.30 | error 97.91262817382812 | self_attn.v_proj\n",
            "[1 / 12] |  time 0.34 | error 8083.791015625 | self_attn.q_proj\n",
            "[1 / 12] |  time 0.32 | error 2.710146427154541 | self_attn.out_proj\n",
            "[1 / 12] |  time 0.38 | error 242.84774780273438 | fc1\n",
            "[1 / 12] |  time 1.28 | error 10.808691024780273 | fc2\n",
            "quantizing activations ...\n",
            "[2 / 12] |  time 0.46 | error 6765.34716796875 | self_attn.k_proj\n",
            "[2 / 12] |  time 0.45 | error 84.13201141357422 | self_attn.v_proj\n",
            "[2 / 12] |  time 0.42 | error 3603.398193359375 | self_attn.q_proj\n",
            "[2 / 12] |  time 0.31 | error 2.763556480407715 | self_attn.out_proj\n",
            "[2 / 12] |  time 0.37 | error 1144.73388671875 | fc1\n",
            "[2 / 12] |  time 1.22 | error 9.255569458007812 | fc2\n",
            "quantizing activations ...\n",
            "[3 / 12] |  time 0.32 | error 12509.92578125 | self_attn.k_proj\n",
            "[3 / 12] |  time 0.32 | error 245.31610107421875 | self_attn.v_proj\n",
            "[3 / 12] |  time 0.31 | error 8946.1865234375 | self_attn.q_proj\n",
            "[3 / 12] |  time 0.32 | error 1.910596251487732 | self_attn.out_proj\n",
            "[3 / 12] |  time 0.38 | error 1029.4010009765625 | fc1\n",
            "[3 / 12] |  time 1.20 | error 13.768155097961426 | fc2\n",
            "quantizing activations ...\n",
            "[4 / 12] |  time 0.31 | error 17389.52734375 | self_attn.k_proj\n",
            "[4 / 12] |  time 0.33 | error 319.5351257324219 | self_attn.v_proj\n",
            "[4 / 12] |  time 0.31 | error 14941.64453125 | self_attn.q_proj\n",
            "[4 / 12] |  time 0.30 | error 2.0757546424865723 | self_attn.out_proj\n",
            "[4 / 12] |  time 0.37 | error 670.9038696289062 | fc1\n",
            "[4 / 12] |  time 1.21 | error 7.102419376373291 | fc2\n",
            "quantizing activations ...\n",
            "[5 / 12] |  time 0.32 | error 18800.099609375 | self_attn.k_proj\n",
            "[5 / 12] |  time 0.31 | error 391.837890625 | self_attn.v_proj\n",
            "[5 / 12] |  time 0.32 | error 14433.0234375 | self_attn.q_proj\n",
            "[5 / 12] |  time 0.31 | error 2.7953455448150635 | self_attn.out_proj\n",
            "[5 / 12] |  time 0.36 | error 1041.7071533203125 | fc1\n",
            "[5 / 12] |  time 1.26 | error 13.707175254821777 | fc2\n",
            "quantizing activations ...\n",
            "[6 / 12] |  time 0.33 | error 18869.923828125 | self_attn.k_proj\n",
            "[6 / 12] |  time 0.32 | error 335.83056640625 | self_attn.v_proj\n",
            "[6 / 12] |  time 0.32 | error 15196.3994140625 | self_attn.q_proj\n",
            "[6 / 12] |  time 0.33 | error 4.56256103515625 | self_attn.out_proj\n",
            "[6 / 12] |  time 0.37 | error 811.8319091796875 | fc1\n",
            "[6 / 12] |  time 1.21 | error 20.433406829833984 | fc2\n",
            "quantizing activations ...\n",
            "[7 / 12] |  time 0.33 | error 20162.578125 | self_attn.k_proj\n",
            "[7 / 12] |  time 0.32 | error 447.8818359375 | self_attn.v_proj\n",
            "[7 / 12] |  time 0.31 | error 13925.78515625 | self_attn.q_proj\n",
            "[7 / 12] |  time 0.33 | error 5.549628257751465 | self_attn.out_proj\n",
            "[7 / 12] |  time 0.37 | error 814.456787109375 | fc1\n",
            "[7 / 12] |  time 1.66 | error 26.34343719482422 | fc2\n",
            "quantizing activations ...\n",
            "[8 / 12] |  time 0.33 | error 17491.384765625 | self_attn.k_proj\n",
            "[8 / 12] |  time 0.30 | error 542.3739013671875 | self_attn.v_proj\n",
            "[8 / 12] |  time 0.31 | error 12267.275390625 | self_attn.q_proj\n",
            "[8 / 12] |  time 0.32 | error 11.14691162109375 | self_attn.out_proj\n",
            "[8 / 12] |  time 0.36 | error 1003.5086059570312 | fc1\n",
            "[8 / 12] |  time 1.21 | error 26.98171615600586 | fc2\n",
            "quantizing activations ...\n",
            "[9 / 12] |  time 0.44 | error 19111.34375 | self_attn.k_proj\n",
            "[9 / 12] |  time 0.40 | error 710.640869140625 | self_attn.v_proj\n",
            "[9 / 12] |  time 0.44 | error 12993.373046875 | self_attn.q_proj\n",
            "[9 / 12] |  time 0.48 | error 19.16356658935547 | self_attn.out_proj\n",
            "[9 / 12] |  time 0.51 | error 1305.7120361328125 | fc1\n",
            "[9 / 12] |  time 1.26 | error 49.10004806518555 | fc2\n",
            "quantizing activations ...\n",
            "[10 / 12] |  time 0.31 | error 15609.498046875 | self_attn.k_proj\n",
            "[10 / 12] |  time 0.31 | error 757.79541015625 | self_attn.v_proj\n",
            "[10 / 12] |  time 0.32 | error 9367.478515625 | self_attn.q_proj\n",
            "[10 / 12] |  time 0.31 | error 34.18034362792969 | self_attn.out_proj\n",
            "[10 / 12] |  time 0.37 | error 1641.237548828125 | fc1\n",
            "[10 / 12] |  time 1.22 | error 56.022098541259766 | fc2\n",
            "quantizing activations ...\n",
            "[11 / 12] |  time 0.48 | error 16953.888671875 | self_attn.k_proj\n",
            "[11 / 12] |  time 0.30 | error 918.8005981445312 | self_attn.v_proj\n",
            "[11 / 12] |  time 0.31 | error 10852.232421875 | self_attn.q_proj\n",
            "[11 / 12] |  time 0.32 | error 19.12511444091797 | self_attn.out_proj\n",
            "[11 / 12] |  time 0.37 | error 1909.841064453125 | fc1\n",
            "[11 / 12] |  time 1.21 | error 104.78924560546875 | fc2\n",
            "quantizing activations ...\n",
            "[12 / 12] |  time 0.32 | error 9039.642578125 | self_attn.k_proj\n",
            "[12 / 12] |  time 0.31 | error 1154.8785400390625 | self_attn.v_proj\n",
            "[12 / 12] |  time 0.31 | error 6119.2421875 | self_attn.q_proj\n",
            "[12 / 12] |  time 0.32 | error 32.997962951660156 | self_attn.out_proj\n",
            "[12 / 12] |  time 0.37 | error 2724.12939453125 | fc1\n",
            "[12 / 12] |  time 1.20 | error 171.06704711914062 | fc2\n",
            "quantizing activations ...\n",
            "86 s\n",
            "/usr/local/lib/python3.10/dist-packages/datasets/load.py:1491: FutureWarning: The repository for ptb_text_only contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/ptb_text_only\n",
            "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
            "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
            "  warnings.warn(\n",
            "ptb\n",
            "Evaluating ...\n",
            "[0 / 12]\n",
            "[1 / 12]\n",
            "[2 / 12]\n",
            "[3 / 12]\n",
            "[4 / 12]\n",
            "[5 / 12]\n",
            "[6 / 12]\n",
            "[7 / 12]\n",
            "[8 / 12]\n",
            "[9 / 12]\n",
            "[10 / 12]\n",
            "[11 / 12]\n",
            "52 PPL\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Packing ...\n",
            "model.decoder.layers.0.self_attn.k_proj\n",
            "model.decoder.layers.0.self_attn.v_proj\n",
            "model.decoder.layers.0.self_attn.q_proj\n",
            "model.decoder.layers.0.self_attn.out_proj\n",
            "model.decoder.layers.0.fc1\n",
            "model.decoder.layers.0.fc2\n",
            "model.decoder.layers.1.self_attn.k_proj\n",
            "model.decoder.layers.1.self_attn.v_proj\n",
            "model.decoder.layers.1.self_attn.q_proj\n",
            "model.decoder.layers.1.self_attn.out_proj\n",
            "model.decoder.layers.1.fc1\n",
            "model.decoder.layers.1.fc2\n",
            "model.decoder.layers.2.self_attn.k_proj\n",
            "model.decoder.layers.2.self_attn.v_proj\n",
            "model.decoder.layers.2.self_attn.q_proj\n",
            "model.decoder.layers.2.self_attn.out_proj\n",
            "model.decoder.layers.2.fc1\n",
            "model.decoder.layers.2.fc2\n",
            "model.decoder.layers.3.self_attn.k_proj\n",
            "model.decoder.layers.3.self_attn.v_proj\n",
            "model.decoder.layers.3.self_attn.q_proj\n",
            "model.decoder.layers.3.self_attn.out_proj\n",
            "model.decoder.layers.3.fc1\n",
            "model.decoder.layers.3.fc2\n",
            "model.decoder.layers.4.self_attn.k_proj\n",
            "model.decoder.layers.4.self_attn.v_proj\n",
            "model.decoder.layers.4.self_attn.q_proj\n",
            "model.decoder.layers.4.self_attn.out_proj\n",
            "model.decoder.layers.4.fc1\n",
            "model.decoder.layers.4.fc2\n",
            "model.decoder.layers.5.self_attn.k_proj\n",
            "model.decoder.layers.5.self_attn.v_proj\n",
            "model.decoder.layers.5.self_attn.q_proj\n",
            "model.decoder.layers.5.self_attn.out_proj\n",
            "model.decoder.layers.5.fc1\n",
            "model.decoder.layers.5.fc2\n",
            "model.decoder.layers.6.self_attn.k_proj\n",
            "model.decoder.layers.6.self_attn.v_proj\n",
            "model.decoder.layers.6.self_attn.q_proj\n",
            "model.decoder.layers.6.self_attn.out_proj\n",
            "model.decoder.layers.6.fc1\n",
            "model.decoder.layers.6.fc2\n",
            "model.decoder.layers.7.self_attn.k_proj\n",
            "model.decoder.layers.7.self_attn.v_proj\n",
            "model.decoder.layers.7.self_attn.q_proj\n",
            "model.decoder.layers.7.self_attn.out_proj\n",
            "model.decoder.layers.7.fc1\n",
            "model.decoder.layers.7.fc2\n",
            "model.decoder.layers.8.self_attn.k_proj\n",
            "model.decoder.layers.8.self_attn.v_proj\n",
            "model.decoder.layers.8.self_attn.q_proj\n",
            "model.decoder.layers.8.self_attn.out_proj\n",
            "model.decoder.layers.8.fc1\n",
            "model.decoder.layers.8.fc2\n",
            "model.decoder.layers.9.self_attn.k_proj\n",
            "model.decoder.layers.9.self_attn.v_proj\n",
            "model.decoder.layers.9.self_attn.q_proj\n",
            "model.decoder.layers.9.self_attn.out_proj\n",
            "model.decoder.layers.9.fc1\n",
            "model.decoder.layers.9.fc2\n",
            "model.decoder.layers.10.self_attn.k_proj\n",
            "model.decoder.layers.10.self_attn.v_proj\n",
            "model.decoder.layers.10.self_attn.q_proj\n",
            "model.decoder.layers.10.self_attn.out_proj\n",
            "model.decoder.layers.10.fc1\n",
            "model.decoder.layers.10.fc2\n",
            "model.decoder.layers.11.self_attn.k_proj\n",
            "model.decoder.layers.11.self_attn.v_proj\n",
            "model.decoder.layers.11.self_attn.q_proj\n",
            "model.decoder.layers.11.self_attn.out_proj\n",
            "model.decoder.layers.11.fc1\n",
            "model.decoder.layers.11.fc2\n",
            "Done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python opt.py facebook/opt-125m c4 --load opt-125m-w4-a4-token-n128.pt --benchmark 128 --check\n"
      ],
      "metadata": {
        "id": "WYu0JVL6pHY7",
        "outputId": "9884cc8b-2203-40f1-e497-35e25ed3bd04",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Loading model ...\n",
            "Done.\n",
            "Processed 256 sequences.\n",
            "Benchmarking ...\n",
            "0 0.6871809959411621\n",
            "1 0.01626300811767578\n",
            "2 0.011291265487670898\n",
            "3 0.010997295379638672\n",
            "4 0.010951757431030273\n",
            "5 0.010896921157836914\n",
            "6 0.010783672332763672\n",
            "7 0.01085805892944336\n",
            "8 0.013136863708496094\n",
            "9 0.01107478141784668\n",
            "10 0.010917901992797852\n",
            "11 0.011011362075805664\n",
            "12 0.011098146438598633\n",
            "13 0.011223316192626953\n",
            "14 0.010944366455078125\n",
            "15 0.011432886123657227\n",
            "16 0.01123666763305664\n",
            "17 0.011410951614379883\n",
            "18 0.01109623908996582\n",
            "19 0.010853052139282227\n",
            "20 0.010886430740356445\n",
            "21 0.011017799377441406\n",
            "22 0.01085662841796875\n",
            "23 0.011025190353393555\n",
            "24 0.011507987976074219\n",
            "25 0.011124372482299805\n",
            "26 0.014913082122802734\n",
            "27 0.010986089706420898\n",
            "28 0.011120319366455078\n",
            "29 0.011007547378540039\n",
            "30 0.012671232223510742\n",
            "31 0.01145172119140625\n",
            "32 0.01127481460571289\n",
            "33 0.01094961166381836\n",
            "34 0.011600017547607422\n",
            "35 0.010906219482421875\n",
            "36 0.011120319366455078\n",
            "37 0.011684894561767578\n",
            "38 0.012819528579711914\n",
            "39 0.011812925338745117\n",
            "40 0.01130819320678711\n",
            "41 0.011188983917236328\n",
            "42 0.012243986129760742\n",
            "43 0.011978387832641602\n",
            "44 0.0112152099609375\n",
            "45 0.011346101760864258\n",
            "46 0.015496015548706055\n",
            "47 0.01127314567565918\n",
            "48 0.01108098030090332\n",
            "49 0.01112985610961914\n",
            "50 0.012009859085083008\n",
            "51 0.015010595321655273\n",
            "52 0.011113166809082031\n",
            "53 0.011515140533447266\n",
            "54 0.010935544967651367\n",
            "55 0.010888338088989258\n",
            "56 0.010877370834350586\n",
            "57 0.011136054992675781\n",
            "58 0.010973930358886719\n",
            "59 0.010897636413574219\n",
            "60 0.01143193244934082\n",
            "61 0.012845754623413086\n",
            "62 0.018609285354614258\n",
            "63 0.011972904205322266\n",
            "64 0.011227607727050781\n",
            "65 0.011805057525634766\n",
            "66 0.01070857048034668\n",
            "67 0.011158466339111328\n",
            "68 0.011065483093261719\n",
            "69 0.013244390487670898\n",
            "70 0.011095285415649414\n",
            "71 0.010903596878051758\n",
            "72 0.010979175567626953\n",
            "73 0.011417388916015625\n",
            "74 0.012932062149047852\n",
            "75 0.012209892272949219\n",
            "76 0.016339540481567383\n",
            "77 0.012371301651000977\n",
            "78 0.011736869812011719\n",
            "79 0.011399030685424805\n",
            "80 0.011127710342407227\n",
            "81 0.014844179153442383\n",
            "82 0.011808633804321289\n",
            "83 0.013151407241821289\n",
            "84 0.013048410415649414\n",
            "85 0.011806726455688477\n",
            "86 0.012447118759155273\n",
            "87 0.01141810417175293\n",
            "88 0.012280464172363281\n",
            "89 0.011056184768676758\n",
            "90 0.010930776596069336\n",
            "91 0.01135563850402832\n",
            "92 0.014983654022216797\n",
            "93 0.01076817512512207\n",
            "94 0.010690927505493164\n",
            "95 0.011093854904174805\n",
            "96 0.010989189147949219\n",
            "97 0.011024951934814453\n",
            "98 0.011426687240600586\n",
            "99 0.01163482666015625\n",
            "100 0.011409282684326172\n",
            "101 0.011644363403320312\n",
            "102 0.011065244674682617\n",
            "103 0.010721206665039062\n",
            "104 0.010727167129516602\n",
            "105 0.010655641555786133\n",
            "106 0.010744571685791016\n",
            "107 0.011002063751220703\n",
            "108 0.01091146469116211\n",
            "109 0.010725736618041992\n",
            "110 0.011069774627685547\n",
            "111 0.010760784149169922\n",
            "112 0.01102757453918457\n",
            "113 0.010711193084716797\n",
            "114 0.011262893676757812\n",
            "115 0.011106252670288086\n",
            "116 0.013768434524536133\n",
            "117 0.010905981063842773\n",
            "118 0.011911869049072266\n",
            "119 0.013082504272460938\n",
            "120 0.011168718338012695\n",
            "121 0.011523962020874023\n",
            "122 0.012858390808105469\n",
            "123 0.011545419692993164\n",
            "124 0.011574745178222656\n",
            "125 0.011448860168457031\n",
            "126 0.011197805404663086\n",
            "127 0.011425256729125977\n",
            "Median: 0.011219263076782227\n",
            "PPL: 14718.673828125\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## W4_A4_REOPTIMIZE\n"
      ],
      "metadata": {
        "id": "RpdUh-m9pFHc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!python opt.py facebook/opt-125m c4 --wbits 4 --nsamples 128 --abits 4 --amethod reoptimize --save opt-125m-w4-a4-reoptimize-n128.pt\n"
      ],
      "metadata": {
        "id": "ZVCEYLf4o936",
        "outputId": "62a6694a-61cf-481c-8f9b-5ed28db1e119",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processed 256 sequences.\n",
            "Starting ...\n",
            "Ready.\n",
            "iterating decoder layers...\n",
            "USING OBS ACTIVATION Q\n",
            "quantizing weights\n",
            "[1 / 12] |  time 0.54 | error 11456.970703125 | self_attn.k_proj\n",
            "[1 / 12] |  time 0.31 | error 97.91262817382812 | self_attn.v_proj\n",
            "[1 / 12] |  time 0.32 | error 8083.791015625 | self_attn.q_proj\n",
            "[1 / 12] |  time 0.32 | error 2.710146427154541 | self_attn.out_proj\n",
            "[1 / 12] |  time 0.38 | error 242.84774780273438 | fc1\n",
            "[1 / 12] |  time 1.28 | error 10.808691024780273 | fc2\n",
            "quantizing activations ...\n",
            "[2 / 12] |  time 0.31 | error 6765.34716796875 | self_attn.k_proj\n",
            "[2 / 12] |  time 0.31 | error 84.13201141357422 | self_attn.v_proj\n",
            "[2 / 12] |  time 0.31 | error 3603.398193359375 | self_attn.q_proj\n",
            "[2 / 12] |  time 0.31 | error 2.763556480407715 | self_attn.out_proj\n",
            "[2 / 12] |  time 0.38 | error 1144.73388671875 | fc1\n",
            "[2 / 12] |  time 1.22 | error 9.255569458007812 | fc2\n",
            "quantizing activations ...\n",
            "[3 / 12] |  time 0.40 | error 12509.92578125 | self_attn.k_proj\n",
            "[3 / 12] |  time 0.30 | error 245.31610107421875 | self_attn.v_proj\n",
            "[3 / 12] |  time 0.31 | error 8946.1865234375 | self_attn.q_proj\n",
            "[3 / 12] |  time 0.31 | error 1.910596251487732 | self_attn.out_proj\n",
            "[3 / 12] |  time 0.37 | error 1029.4010009765625 | fc1\n",
            "[3 / 12] |  time 1.23 | error 13.768155097961426 | fc2\n",
            "quantizing activations ...\n",
            "[4 / 12] |  time 0.32 | error 17389.52734375 | self_attn.k_proj\n",
            "[4 / 12] |  time 0.31 | error 319.5351257324219 | self_attn.v_proj\n",
            "[4 / 12] |  time 0.31 | error 14941.64453125 | self_attn.q_proj\n",
            "[4 / 12] |  time 0.32 | error 2.0757546424865723 | self_attn.out_proj\n",
            "[4 / 12] |  time 0.37 | error 670.9038696289062 | fc1\n",
            "[4 / 12] |  time 1.22 | error 7.102419376373291 | fc2\n",
            "quantizing activations ...\n",
            "[5 / 12] |  time 0.52 | error 18800.099609375 | self_attn.k_proj\n",
            "[5 / 12] |  time 0.32 | error 391.837890625 | self_attn.v_proj\n",
            "[5 / 12] |  time 0.31 | error 14433.0234375 | self_attn.q_proj\n",
            "[5 / 12] |  time 0.32 | error 2.7953455448150635 | self_attn.out_proj\n",
            "[5 / 12] |  time 0.37 | error 1041.7071533203125 | fc1\n",
            "[5 / 12] |  time 1.21 | error 13.707175254821777 | fc2\n",
            "quantizing activations ...\n",
            "[6 / 12] |  time 0.32 | error 18869.923828125 | self_attn.k_proj\n",
            "[6 / 12] |  time 0.32 | error 335.83056640625 | self_attn.v_proj\n",
            "[6 / 12] |  time 0.33 | error 15196.3994140625 | self_attn.q_proj\n",
            "[6 / 12] |  time 0.32 | error 4.56256103515625 | self_attn.out_proj\n",
            "[6 / 12] |  time 0.37 | error 811.8319091796875 | fc1\n",
            "[6 / 12] |  time 1.23 | error 20.433406829833984 | fc2\n",
            "quantizing activations ...\n",
            "[7 / 12] |  time 0.47 | error 20162.578125 | self_attn.k_proj\n",
            "[7 / 12] |  time 0.37 | error 447.8818359375 | self_attn.v_proj\n",
            "[7 / 12] |  time 0.32 | error 13925.78515625 | self_attn.q_proj\n",
            "[7 / 12] |  time 0.31 | error 5.549628257751465 | self_attn.out_proj\n",
            "[7 / 12] |  time 0.38 | error 814.456787109375 | fc1\n",
            "[7 / 12] |  time 1.21 | error 26.34343719482422 | fc2\n",
            "quantizing activations ...\n",
            "[8 / 12] |  time 0.42 | error 17491.384765625 | self_attn.k_proj\n",
            "[8 / 12] |  time 0.40 | error 542.3739013671875 | self_attn.v_proj\n",
            "[8 / 12] |  time 0.44 | error 12267.275390625 | self_attn.q_proj\n",
            "[8 / 12] |  time 0.47 | error 11.14691162109375 | self_attn.out_proj\n",
            "[8 / 12] |  time 0.53 | error 1003.5086059570312 | fc1\n",
            "[8 / 12] |  time 1.28 | error 26.98171615600586 | fc2\n",
            "quantizing activations ...\n",
            "[9 / 12] |  time 0.33 | error 19111.34375 | self_attn.k_proj\n",
            "[9 / 12] |  time 0.32 | error 710.640869140625 | self_attn.v_proj\n",
            "[9 / 12] |  time 0.31 | error 12993.373046875 | self_attn.q_proj\n",
            "[9 / 12] |  time 0.32 | error 19.16356658935547 | self_attn.out_proj\n",
            "[9 / 12] |  time 0.37 | error 1305.7120361328125 | fc1\n",
            "[9 / 12] |  time 1.20 | error 49.10004806518555 | fc2\n",
            "quantizing activations ...\n",
            "[10 / 12] |  time 0.33 | error 15609.498046875 | self_attn.k_proj\n",
            "[10 / 12] |  time 0.31 | error 757.79541015625 | self_attn.v_proj\n",
            "[10 / 12] |  time 0.32 | error 9367.478515625 | self_attn.q_proj\n",
            "[10 / 12] |  time 0.32 | error 34.18034362792969 | self_attn.out_proj\n",
            "[10 / 12] |  time 0.37 | error 1641.237548828125 | fc1\n",
            "[10 / 12] |  time 1.24 | error 56.022098541259766 | fc2\n",
            "quantizing activations ...\n",
            "[11 / 12] |  time 0.34 | error 16953.888671875 | self_attn.k_proj\n",
            "[11 / 12] |  time 0.30 | error 918.8005981445312 | self_attn.v_proj\n",
            "[11 / 12] |  time 0.34 | error 10852.232421875 | self_attn.q_proj\n",
            "[11 / 12] |  time 0.32 | error 19.12511444091797 | self_attn.out_proj\n",
            "[11 / 12] |  time 0.37 | error 1909.841064453125 | fc1\n",
            "[11 / 12] |  time 1.28 | error 104.78924560546875 | fc2\n",
            "quantizing activations ...\n",
            "[12 / 12] |  time 0.32 | error 9039.642578125 | self_attn.k_proj\n",
            "[12 / 12] |  time 0.32 | error 1154.8785400390625 | self_attn.v_proj\n",
            "[12 / 12] |  time 0.33 | error 6119.2421875 | self_attn.q_proj\n",
            "[12 / 12] |  time 0.32 | error 32.997962951660156 | self_attn.out_proj\n",
            "[12 / 12] |  time 0.39 | error 2724.12939453125 | fc1\n",
            "[12 / 12] |  time 1.22 | error 171.06704711914062 | fc2\n",
            "quantizing activations ...\n",
            "77 s\n",
            "/usr/local/lib/python3.10/dist-packages/datasets/load.py:1491: FutureWarning: The repository for ptb_text_only contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/ptb_text_only\n",
            "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
            "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
            "  warnings.warn(\n",
            "ptb\n",
            "Evaluating ...\n",
            "[0 / 12]\n",
            "[1 / 12]\n",
            "[2 / 12]\n",
            "[3 / 12]\n",
            "[4 / 12]\n",
            "[5 / 12]\n",
            "[6 / 12]\n",
            "[7 / 12]\n",
            "[8 / 12]\n",
            "[9 / 12]\n",
            "[10 / 12]\n",
            "[11 / 12]\n",
            "52 PPL\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Packing ...\n",
            "model.decoder.layers.0.self_attn.k_proj\n",
            "model.decoder.layers.0.self_attn.v_proj\n",
            "model.decoder.layers.0.self_attn.q_proj\n",
            "model.decoder.layers.0.self_attn.out_proj\n",
            "model.decoder.layers.0.fc1\n",
            "model.decoder.layers.0.fc2\n",
            "model.decoder.layers.1.self_attn.k_proj\n",
            "model.decoder.layers.1.self_attn.v_proj\n",
            "model.decoder.layers.1.self_attn.q_proj\n",
            "model.decoder.layers.1.self_attn.out_proj\n",
            "model.decoder.layers.1.fc1\n",
            "model.decoder.layers.1.fc2\n",
            "model.decoder.layers.2.self_attn.k_proj\n",
            "model.decoder.layers.2.self_attn.v_proj\n",
            "model.decoder.layers.2.self_attn.q_proj\n",
            "model.decoder.layers.2.self_attn.out_proj\n",
            "model.decoder.layers.2.fc1\n",
            "model.decoder.layers.2.fc2\n",
            "model.decoder.layers.3.self_attn.k_proj\n",
            "model.decoder.layers.3.self_attn.v_proj\n",
            "model.decoder.layers.3.self_attn.q_proj\n",
            "model.decoder.layers.3.self_attn.out_proj\n",
            "model.decoder.layers.3.fc1\n",
            "model.decoder.layers.3.fc2\n",
            "model.decoder.layers.4.self_attn.k_proj\n",
            "model.decoder.layers.4.self_attn.v_proj\n",
            "model.decoder.layers.4.self_attn.q_proj\n",
            "model.decoder.layers.4.self_attn.out_proj\n",
            "model.decoder.layers.4.fc1\n",
            "model.decoder.layers.4.fc2\n",
            "model.decoder.layers.5.self_attn.k_proj\n",
            "model.decoder.layers.5.self_attn.v_proj\n",
            "model.decoder.layers.5.self_attn.q_proj\n",
            "model.decoder.layers.5.self_attn.out_proj\n",
            "model.decoder.layers.5.fc1\n",
            "model.decoder.layers.5.fc2\n",
            "model.decoder.layers.6.self_attn.k_proj\n",
            "model.decoder.layers.6.self_attn.v_proj\n",
            "model.decoder.layers.6.self_attn.q_proj\n",
            "model.decoder.layers.6.self_attn.out_proj\n",
            "model.decoder.layers.6.fc1\n",
            "model.decoder.layers.6.fc2\n",
            "model.decoder.layers.7.self_attn.k_proj\n",
            "model.decoder.layers.7.self_attn.v_proj\n",
            "model.decoder.layers.7.self_attn.q_proj\n",
            "model.decoder.layers.7.self_attn.out_proj\n",
            "model.decoder.layers.7.fc1\n",
            "model.decoder.layers.7.fc2\n",
            "model.decoder.layers.8.self_attn.k_proj\n",
            "model.decoder.layers.8.self_attn.v_proj\n",
            "model.decoder.layers.8.self_attn.q_proj\n",
            "model.decoder.layers.8.self_attn.out_proj\n",
            "model.decoder.layers.8.fc1\n",
            "model.decoder.layers.8.fc2\n",
            "model.decoder.layers.9.self_attn.k_proj\n",
            "model.decoder.layers.9.self_attn.v_proj\n",
            "model.decoder.layers.9.self_attn.q_proj\n",
            "model.decoder.layers.9.self_attn.out_proj\n",
            "model.decoder.layers.9.fc1\n",
            "model.decoder.layers.9.fc2\n",
            "model.decoder.layers.10.self_attn.k_proj\n",
            "model.decoder.layers.10.self_attn.v_proj\n",
            "model.decoder.layers.10.self_attn.q_proj\n",
            "model.decoder.layers.10.self_attn.out_proj\n",
            "model.decoder.layers.10.fc1\n",
            "model.decoder.layers.10.fc2\n",
            "model.decoder.layers.11.self_attn.k_proj\n",
            "model.decoder.layers.11.self_attn.v_proj\n",
            "model.decoder.layers.11.self_attn.q_proj\n",
            "model.decoder.layers.11.self_attn.out_proj\n",
            "model.decoder.layers.11.fc1\n",
            "model.decoder.layers.11.fc2\n",
            "Done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python opt.py facebook/opt-125m c4 --load opt-125m-w4-a4-reoptimize-n128.pt --benchmark 128 --check\n"
      ],
      "metadata": {
        "id": "82GM91R6pIP1",
        "outputId": "2edfcc10-12aa-4fff-fc73-be0052387b29",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Loading model ...\n",
            "Done.\n",
            "Processed 256 sequences.\n",
            "Benchmarking ...\n",
            "0 0.7190148830413818\n",
            "1 0.01622605323791504\n",
            "2 0.01186227798461914\n",
            "3 0.01449131965637207\n",
            "4 0.014016866683959961\n",
            "5 0.011894464492797852\n",
            "6 0.011161327362060547\n",
            "7 0.011173725128173828\n",
            "8 0.012731075286865234\n",
            "9 0.012440681457519531\n",
            "10 0.011213064193725586\n",
            "11 0.01144552230834961\n",
            "12 0.011149883270263672\n",
            "13 0.011862516403198242\n",
            "14 0.010955810546875\n",
            "15 0.011220932006835938\n",
            "16 0.011001348495483398\n",
            "17 0.011086702346801758\n",
            "18 0.01303720474243164\n",
            "19 0.013787269592285156\n",
            "20 0.011067628860473633\n",
            "21 0.011035442352294922\n",
            "22 0.010921478271484375\n",
            "23 0.011243104934692383\n",
            "24 0.013065099716186523\n",
            "25 0.011383295059204102\n",
            "26 0.011098384857177734\n",
            "27 0.012705326080322266\n",
            "28 0.01144719123840332\n",
            "29 0.011090517044067383\n",
            "30 0.011721372604370117\n",
            "31 0.01123952865600586\n",
            "32 0.01148843765258789\n",
            "33 0.011413097381591797\n",
            "34 0.011251688003540039\n",
            "35 0.011489629745483398\n",
            "36 0.011360406875610352\n",
            "37 0.014228582382202148\n",
            "38 0.01124119758605957\n",
            "39 0.011379480361938477\n",
            "40 0.014078378677368164\n",
            "41 0.01907062530517578\n",
            "42 0.011558771133422852\n",
            "43 0.011394023895263672\n",
            "44 0.011698722839355469\n",
            "45 0.011171579360961914\n",
            "46 0.011173009872436523\n",
            "47 0.011406898498535156\n",
            "48 0.01143503189086914\n",
            "49 0.011242866516113281\n",
            "50 0.014573335647583008\n",
            "51 0.011816978454589844\n",
            "52 0.011661767959594727\n",
            "53 0.011793851852416992\n",
            "54 0.011429309844970703\n",
            "55 0.011495113372802734\n",
            "56 0.011089563369750977\n",
            "57 0.011265754699707031\n",
            "58 0.011866569519042969\n",
            "59 0.011429309844970703\n",
            "60 0.011494874954223633\n",
            "61 0.01434636116027832\n",
            "62 0.011336088180541992\n",
            "63 0.011353254318237305\n",
            "64 0.011075973510742188\n",
            "65 0.0108184814453125\n",
            "66 0.010908365249633789\n",
            "67 0.011869192123413086\n",
            "68 0.01108098030090332\n",
            "69 0.011925697326660156\n",
            "70 0.011405229568481445\n",
            "71 0.010760068893432617\n",
            "72 0.011066198348999023\n",
            "73 0.010917901992797852\n",
            "74 0.010958194732666016\n",
            "75 0.013085603713989258\n",
            "76 0.01297450065612793\n",
            "77 0.014548778533935547\n",
            "78 0.013199090957641602\n",
            "79 0.012334108352661133\n",
            "80 0.011774539947509766\n",
            "81 0.011974573135375977\n",
            "82 0.011380195617675781\n",
            "83 0.012301206588745117\n",
            "84 0.013115644454956055\n",
            "85 0.012006044387817383\n",
            "86 0.01410984992980957\n",
            "87 0.012403488159179688\n",
            "88 0.011388540267944336\n",
            "89 0.011495351791381836\n",
            "90 0.011718988418579102\n",
            "91 0.012680768966674805\n",
            "92 0.011954545974731445\n",
            "93 0.011106729507446289\n",
            "94 0.011726617813110352\n",
            "95 0.011228084564208984\n",
            "96 0.010937213897705078\n",
            "97 0.010959386825561523\n",
            "98 0.010721206665039062\n",
            "99 0.010835409164428711\n",
            "100 0.012972831726074219\n",
            "101 0.01342320442199707\n",
            "102 0.011196613311767578\n",
            "103 0.01101541519165039\n",
            "104 0.01077413558959961\n",
            "105 0.010638236999511719\n",
            "106 0.011009454727172852\n",
            "107 0.011050939559936523\n",
            "108 0.010745763778686523\n",
            "109 0.010694742202758789\n",
            "110 0.012819290161132812\n",
            "111 0.011644840240478516\n",
            "112 0.01983356475830078\n",
            "113 0.015491724014282227\n",
            "114 0.013884782791137695\n",
            "115 0.011753082275390625\n",
            "116 0.011690855026245117\n",
            "117 0.011922836303710938\n",
            "118 0.011661052703857422\n",
            "119 0.01200413703918457\n",
            "120 0.01189422607421875\n",
            "121 0.011501789093017578\n",
            "122 0.014600038528442383\n",
            "123 0.011652469635009766\n",
            "124 0.015743255615234375\n",
            "125 0.017859697341918945\n",
            "126 0.01226663589477539\n",
            "127 0.011627912521362305\n",
            "Median: 0.011495232582092285\n",
            "PPL: 14719.1240234375\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## W4_A4_CLE\n"
      ],
      "metadata": {
        "id": "MmNDG6CSpIu0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!python opt.py facebook/opt-125m c4 --wbits 4 --abits 4 --nsamples 128 --cle 1 --save opt-125m-w4-a4-cle-n128.pt\n"
      ],
      "metadata": {
        "id": "lt1vCj-fo-2e",
        "outputId": "db3120fb-42dd-4563-9513-28d140c96c99",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processed 256 sequences.\n",
            "Starting ...\n",
            "Ready.\n",
            "iterating decoder layers...\n",
            "USING OBS ACTIVATION Q\n",
            "quantizing weights\n",
            "[1 / 12] |  time 0.78 | error 2778.73681640625 | self_attn.k_proj\n",
            "[1 / 12] |  time 0.47 | error 854.239501953125 | self_attn.v_proj\n",
            "[1 / 12] |  time 0.40 | error 5395.6357421875 | self_attn.q_proj\n",
            "[1 / 12] |  time 0.30 | error 3.012087821960449 | self_attn.out_proj\n",
            "[1 / 12] |  time 0.39 | error 123.57704162597656 | fc1\n",
            "[1 / 12] |  time 1.25 | error 4.977382659912109 | fc2\n",
            "quantizing activations ...\n",
            "[2 / 12] |  time 0.31 | error 1045.382568359375 | self_attn.k_proj\n",
            "[2 / 12] |  time 0.30 | error 481.5646057128906 | self_attn.v_proj\n",
            "[2 / 12] |  time 0.30 | error 2823.690185546875 | self_attn.q_proj\n",
            "[2 / 12] |  time 0.31 | error 2.4591166973114014 | self_attn.out_proj\n",
            "[2 / 12] |  time 0.38 | error 1350.5838623046875 | fc1\n",
            "[2 / 12] |  time 1.18 | error 7.787049770355225 | fc2\n",
            "quantizing activations ...\n",
            "[3 / 12] |  time 0.46 | error 1279.6695556640625 | self_attn.k_proj\n",
            "[3 / 12] |  time 0.47 | error 1121.94580078125 | self_attn.v_proj\n",
            "[3 / 12] |  time 0.47 | error 2129.7822265625 | self_attn.q_proj\n",
            "[3 / 12] |  time 0.31 | error 16.736919403076172 | self_attn.out_proj\n",
            "[3 / 12] |  time 0.37 | error 222.72300720214844 | fc1\n",
            "[3 / 12] |  time 1.21 | error 1.9769057035446167 | fc2\n",
            "quantizing activations ...\n",
            "[4 / 12] |  time 0.32 | error 1299.2197265625 | self_attn.k_proj\n",
            "[4 / 12] |  time 0.31 | error 193.87071228027344 | self_attn.v_proj\n",
            "[4 / 12] |  time 0.31 | error 1462.826904296875 | self_attn.q_proj\n",
            "[4 / 12] |  time 0.31 | error 7.303255081176758 | self_attn.out_proj\n",
            "[4 / 12] |  time 0.38 | error 132.62954711914062 | fc1\n",
            "[4 / 12] |  time 1.17 | error 1.1826558113098145 | fc2\n",
            "quantizing activations ...\n",
            "[5 / 12] |  time 0.44 | error 1153.605224609375 | self_attn.k_proj\n",
            "[5 / 12] |  time 0.50 | error 214.75665283203125 | self_attn.v_proj\n",
            "[5 / 12] |  time 0.46 | error 995.4599609375 | self_attn.q_proj\n",
            "[5 / 12] |  time 0.44 | error 24.245725631713867 | self_attn.out_proj\n",
            "[5 / 12] |  time 0.36 | error 96.25282287597656 | fc1\n",
            "[5 / 12] |  time 1.21 | error 0.8286391496658325 | fc2\n",
            "quantizing activations ...\n",
            "[6 / 12] |  time 0.32 | error 317.46612548828125 | self_attn.k_proj\n",
            "[6 / 12] |  time 0.31 | error 352.2181396484375 | self_attn.v_proj\n",
            "[6 / 12] |  time 0.31 | error 273.205322265625 | self_attn.q_proj\n",
            "[6 / 12] |  time 0.32 | error 127.1017074584961 | self_attn.out_proj\n",
            "[6 / 12] |  time 0.36 | error 45.30931091308594 | fc1\n",
            "[6 / 12] |  time 1.20 | error 0.9335471391677856 | fc2\n",
            "quantizing activations ...\n",
            "[7 / 12] |  time 0.44 | error 164.976806640625 | self_attn.k_proj\n",
            "[7 / 12] |  time 0.46 | error 206.06068420410156 | self_attn.v_proj\n",
            "[7 / 12] |  time 0.48 | error 228.48573303222656 | self_attn.q_proj\n",
            "[7 / 12] |  time 0.43 | error 109.86384582519531 | self_attn.out_proj\n",
            "[7 / 12] |  time 0.37 | error 65.78901672363281 | fc1\n",
            "[7 / 12] |  time 1.23 | error 1.1018732786178589 | fc2\n",
            "quantizing activations ...\n",
            "[8 / 12] |  time 0.31 | error 106.012451171875 | self_attn.k_proj\n",
            "[8 / 12] |  time 0.31 | error 232.07461547851562 | self_attn.v_proj\n",
            "[8 / 12] |  time 0.31 | error 121.40861511230469 | self_attn.q_proj\n",
            "[8 / 12] |  time 0.30 | error 265.0511474609375 | self_attn.out_proj\n",
            "[8 / 12] |  time 0.37 | error 65.89734649658203 | fc1\n",
            "[8 / 12] |  time 1.22 | error 0.8058255910873413 | fc2\n",
            "quantizing activations ...\n",
            "[9 / 12] |  time 0.43 | error 184.8526153564453 | self_attn.k_proj\n",
            "[9 / 12] |  time 0.47 | error 117.49838256835938 | self_attn.v_proj\n",
            "[9 / 12] |  time 0.50 | error 207.40615844726562 | self_attn.q_proj\n",
            "[9 / 12] |  time 0.47 | error 57.925514221191406 | self_attn.out_proj\n",
            "[9 / 12] |  time 0.38 | error 93.919189453125 | fc1\n",
            "[9 / 12] |  time 1.21 | error 2.323619842529297 | fc2\n",
            "quantizing activations ...\n",
            "[10 / 12] |  time 0.31 | error 198.96754455566406 | self_attn.k_proj\n",
            "[10 / 12] |  time 0.32 | error 137.94680786132812 | self_attn.v_proj\n",
            "[10 / 12] |  time 0.31 | error 204.16668701171875 | self_attn.q_proj\n",
            "[10 / 12] |  time 0.31 | error 70.43637084960938 | self_attn.out_proj\n",
            "[10 / 12] |  time 0.37 | error 38.339908599853516 | fc1\n",
            "[10 / 12] |  time 1.20 | error 3.6083977222442627 | fc2\n",
            "quantizing activations ...\n",
            "[11 / 12] |  time 0.41 | error 334.443603515625 | self_attn.k_proj\n",
            "[11 / 12] |  time 0.48 | error 130.72857666015625 | self_attn.v_proj\n",
            "[11 / 12] |  time 0.46 | error 189.74407958984375 | self_attn.q_proj\n",
            "[11 / 12] |  time 0.46 | error 88.3185043334961 | self_attn.out_proj\n",
            "[11 / 12] |  time 0.40 | error 120.15911865234375 | fc1\n",
            "[11 / 12] |  time 1.23 | error 3.319108724594116 | fc2\n",
            "quantizing activations ...\n",
            "[12 / 12] |  time 0.33 | error 440.9690246582031 | self_attn.k_proj\n",
            "[12 / 12] |  time 0.30 | error 66.60034942626953 | self_attn.v_proj\n",
            "[12 / 12] |  time 0.31 | error 613.1421508789062 | self_attn.q_proj\n",
            "[12 / 12] |  time 0.32 | error 10.16918659210205 | self_attn.out_proj\n",
            "[12 / 12] |  time 0.36 | error 270.9554443359375 | fc1\n",
            "[12 / 12] |  time 1.22 | error 11.168766021728516 | fc2\n",
            "quantizing activations ...\n",
            "77 s\n",
            "/usr/local/lib/python3.10/dist-packages/datasets/load.py:1491: FutureWarning: The repository for ptb_text_only contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/ptb_text_only\n",
            "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
            "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
            "  warnings.warn(\n",
            "ptb\n",
            "Evaluating ...\n",
            "[0 / 12]\n",
            "[1 / 12]\n",
            "[2 / 12]\n",
            "[3 / 12]\n",
            "[4 / 12]\n",
            "[5 / 12]\n",
            "[6 / 12]\n",
            "[7 / 12]\n",
            "[8 / 12]\n",
            "[9 / 12]\n",
            "[10 / 12]\n",
            "[11 / 12]\n",
            "10077 PPL\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Packing ...\n",
            "model.decoder.layers.0.self_attn.k_proj\n",
            "model.decoder.layers.0.self_attn.v_proj\n",
            "model.decoder.layers.0.self_attn.q_proj\n",
            "model.decoder.layers.0.self_attn.out_proj\n",
            "model.decoder.layers.0.fc1\n",
            "model.decoder.layers.0.fc2\n",
            "model.decoder.layers.1.self_attn.k_proj\n",
            "model.decoder.layers.1.self_attn.v_proj\n",
            "model.decoder.layers.1.self_attn.q_proj\n",
            "model.decoder.layers.1.self_attn.out_proj\n",
            "model.decoder.layers.1.fc1\n",
            "model.decoder.layers.1.fc2\n",
            "model.decoder.layers.2.self_attn.k_proj\n",
            "model.decoder.layers.2.self_attn.v_proj\n",
            "model.decoder.layers.2.self_attn.q_proj\n",
            "model.decoder.layers.2.self_attn.out_proj\n",
            "model.decoder.layers.2.fc1\n",
            "model.decoder.layers.2.fc2\n",
            "model.decoder.layers.3.self_attn.k_proj\n",
            "model.decoder.layers.3.self_attn.v_proj\n",
            "model.decoder.layers.3.self_attn.q_proj\n",
            "model.decoder.layers.3.self_attn.out_proj\n",
            "model.decoder.layers.3.fc1\n",
            "model.decoder.layers.3.fc2\n",
            "model.decoder.layers.4.self_attn.k_proj\n",
            "model.decoder.layers.4.self_attn.v_proj\n",
            "model.decoder.layers.4.self_attn.q_proj\n",
            "model.decoder.layers.4.self_attn.out_proj\n",
            "model.decoder.layers.4.fc1\n",
            "model.decoder.layers.4.fc2\n",
            "model.decoder.layers.5.self_attn.k_proj\n",
            "model.decoder.layers.5.self_attn.v_proj\n",
            "model.decoder.layers.5.self_attn.q_proj\n",
            "model.decoder.layers.5.self_attn.out_proj\n",
            "model.decoder.layers.5.fc1\n",
            "model.decoder.layers.5.fc2\n",
            "model.decoder.layers.6.self_attn.k_proj\n",
            "model.decoder.layers.6.self_attn.v_proj\n",
            "model.decoder.layers.6.self_attn.q_proj\n",
            "model.decoder.layers.6.self_attn.out_proj\n",
            "model.decoder.layers.6.fc1\n",
            "model.decoder.layers.6.fc2\n",
            "model.decoder.layers.7.self_attn.k_proj\n",
            "model.decoder.layers.7.self_attn.v_proj\n",
            "model.decoder.layers.7.self_attn.q_proj\n",
            "model.decoder.layers.7.self_attn.out_proj\n",
            "model.decoder.layers.7.fc1\n",
            "model.decoder.layers.7.fc2\n",
            "model.decoder.layers.8.self_attn.k_proj\n",
            "model.decoder.layers.8.self_attn.v_proj\n",
            "model.decoder.layers.8.self_attn.q_proj\n",
            "model.decoder.layers.8.self_attn.out_proj\n",
            "model.decoder.layers.8.fc1\n",
            "model.decoder.layers.8.fc2\n",
            "model.decoder.layers.9.self_attn.k_proj\n",
            "model.decoder.layers.9.self_attn.v_proj\n",
            "model.decoder.layers.9.self_attn.q_proj\n",
            "model.decoder.layers.9.self_attn.out_proj\n",
            "model.decoder.layers.9.fc1\n",
            "model.decoder.layers.9.fc2\n",
            "model.decoder.layers.10.self_attn.k_proj\n",
            "model.decoder.layers.10.self_attn.v_proj\n",
            "model.decoder.layers.10.self_attn.q_proj\n",
            "model.decoder.layers.10.self_attn.out_proj\n",
            "model.decoder.layers.10.fc1\n",
            "model.decoder.layers.10.fc2\n",
            "model.decoder.layers.11.self_attn.k_proj\n",
            "model.decoder.layers.11.self_attn.v_proj\n",
            "model.decoder.layers.11.self_attn.q_proj\n",
            "model.decoder.layers.11.self_attn.out_proj\n",
            "model.decoder.layers.11.fc1\n",
            "model.decoder.layers.11.fc2\n",
            "Done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python opt.py facebook/opt-125m c4 --load opt-125m-w4-a4-cle-n128.pt --benchmark 128 --check\n"
      ],
      "metadata": {
        "id": "NGYGuheQpKFf",
        "outputId": "b4ff44d1-d4b6-4e57-d855-e2a8448cdb78",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Loading model ...\n",
            "Done.\n",
            "Processed 256 sequences.\n",
            "Benchmarking ...\n",
            "0 0.7081587314605713\n",
            "1 0.01962900161743164\n",
            "2 0.011601448059082031\n",
            "3 0.012134552001953125\n",
            "4 0.011565685272216797\n",
            "5 0.011307477951049805\n",
            "6 0.011634588241577148\n",
            "7 0.011422395706176758\n",
            "8 0.011501312255859375\n",
            "9 0.01142430305480957\n",
            "10 0.01154327392578125\n",
            "11 0.011298894882202148\n",
            "12 0.011785507202148438\n",
            "13 0.011604547500610352\n",
            "14 0.011204242706298828\n",
            "15 0.011760473251342773\n",
            "16 0.011117696762084961\n",
            "17 0.011089324951171875\n",
            "18 0.011011362075805664\n",
            "19 0.011563539505004883\n",
            "20 0.013832330703735352\n",
            "21 0.011910676956176758\n",
            "22 0.011616945266723633\n",
            "23 0.010933876037597656\n",
            "24 0.011326789855957031\n",
            "25 0.011856317520141602\n",
            "26 0.011368989944458008\n",
            "27 0.01134634017944336\n",
            "28 0.011586189270019531\n",
            "29 0.012605428695678711\n",
            "30 0.011297941207885742\n",
            "31 0.012003183364868164\n",
            "32 0.011503934860229492\n",
            "33 0.011392354965209961\n",
            "34 0.011243104934692383\n",
            "35 0.01151275634765625\n",
            "36 0.011383056640625\n",
            "37 0.011846542358398438\n",
            "38 0.01140904426574707\n",
            "39 0.011733055114746094\n",
            "40 0.012555360794067383\n",
            "41 0.012533426284790039\n",
            "42 0.011286735534667969\n",
            "43 0.011257410049438477\n",
            "44 0.011441707611083984\n",
            "45 0.011405467987060547\n",
            "46 0.015186071395874023\n",
            "47 0.012558460235595703\n",
            "48 0.011152267456054688\n",
            "49 0.01121664047241211\n",
            "50 0.012774944305419922\n",
            "51 0.011556148529052734\n",
            "52 0.011293888092041016\n",
            "53 0.011638402938842773\n",
            "54 0.013454675674438477\n",
            "55 0.011490583419799805\n",
            "56 0.011507749557495117\n",
            "57 0.011144876480102539\n",
            "58 0.01125192642211914\n",
            "59 0.011358261108398438\n",
            "60 0.011286020278930664\n",
            "61 0.011341094970703125\n",
            "62 0.011501312255859375\n",
            "63 0.011462688446044922\n",
            "64 0.011350393295288086\n",
            "65 0.012230157852172852\n",
            "66 0.011622190475463867\n",
            "67 0.010946035385131836\n",
            "68 0.01121377944946289\n",
            "69 0.011066436767578125\n",
            "70 0.011236906051635742\n",
            "71 0.015076398849487305\n",
            "72 0.013197183609008789\n",
            "73 0.011179685592651367\n",
            "74 0.013210296630859375\n",
            "75 0.012007474899291992\n",
            "76 0.01119852066040039\n",
            "77 0.011323213577270508\n",
            "78 0.011300325393676758\n",
            "79 0.012010812759399414\n",
            "80 0.011546134948730469\n",
            "81 0.011348247528076172\n",
            "82 0.01136636734008789\n",
            "83 0.011493682861328125\n",
            "84 0.012540102005004883\n",
            "85 0.019362688064575195\n",
            "86 0.012954235076904297\n",
            "87 0.011614561080932617\n",
            "88 0.011385679244995117\n",
            "89 0.011229515075683594\n",
            "90 0.013033628463745117\n",
            "91 0.01112222671508789\n",
            "92 0.011206388473510742\n",
            "93 0.011063098907470703\n",
            "94 0.011267662048339844\n",
            "95 0.012302875518798828\n",
            "96 0.012810230255126953\n",
            "97 0.011047840118408203\n",
            "98 0.011272430419921875\n",
            "99 0.011224031448364258\n",
            "100 0.011677265167236328\n",
            "101 0.011152982711791992\n",
            "102 0.011464357376098633\n",
            "103 0.011168479919433594\n",
            "104 0.011503219604492188\n",
            "105 0.011174917221069336\n",
            "106 0.011513471603393555\n",
            "107 0.011200904846191406\n",
            "108 0.011402130126953125\n",
            "109 0.01132345199584961\n",
            "110 0.011073112487792969\n",
            "111 0.011318206787109375\n",
            "112 0.01094365119934082\n",
            "113 0.012393474578857422\n",
            "114 0.011316776275634766\n",
            "115 0.011664390563964844\n",
            "116 0.011483192443847656\n",
            "117 0.01260685920715332\n",
            "118 0.016060352325439453\n",
            "119 0.012143135070800781\n",
            "120 0.0120697021484375\n",
            "121 0.0161440372467041\n",
            "122 0.012546777725219727\n",
            "123 0.012110710144042969\n",
            "124 0.012026071548461914\n",
            "125 0.013735294342041016\n",
            "126 0.011942148208618164\n",
            "127 0.012138605117797852\n",
            "Median: 0.01149749755859375\n",
            "PPL: 14043.5732421875\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## W4_A4_EIG\n"
      ],
      "metadata": {
        "id": "YckjppwXpLBb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!python opt.py facebook/opt-125m c4 --wbits 4 --abits 4 --nsamples 128 --eigenvalues 1 --save opt-125m-w4-a4-eig-n128.pt\n"
      ],
      "metadata": {
        "id": "XJQ8nJH3o_6z",
        "outputId": "a19573f9-a97e-4d35-b46a-0ba8244c8a46",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processed 256 sequences.\n",
            "Starting ...\n",
            "Ready.\n",
            "iterating decoder layers...\n",
            "USING OBS ACTIVATION Q\n",
            "quantizing weights\n",
            "[1 / 12] |  time 0.59 | error 25699.59375 | self_attn.k_proj\n",
            "[1 / 12] |  time 0.33 | error 196.69696044921875 | self_attn.v_proj\n",
            "[1 / 12] |  time 0.33 | error 18219.3515625 | self_attn.q_proj\n",
            "[1 / 12] |  time 0.33 | error 3.203965663909912 | self_attn.out_proj\n",
            "[1 / 12] |  time 0.41 | error 252.86203002929688 | fc1\n",
            "[1 / 12] |  time 1.52 | error 431.032470703125 | fc2\n",
            "quantizing activations ...\n",
            "[2 / 12] |  time 0.35 | error 25806.57421875 | self_attn.k_proj\n",
            "[2 / 12] |  time 0.42 | error 120.58409881591797 | self_attn.v_proj\n",
            "[2 / 12] |  time 0.46 | error 10014.92578125 | self_attn.q_proj\n",
            "[2 / 12] |  time 0.44 | error 4.692255020141602 | self_attn.out_proj\n",
            "[2 / 12] |  time 0.55 | error 1314.8114013671875 | fc1\n",
            "[2 / 12] |  time 1.68 | error 10.216139793395996 | fc2\n",
            "quantizing activations ...\n",
            "[3 / 12] |  time 0.35 | error 131956.625 | self_attn.k_proj\n",
            "[3 / 12] |  time 0.32 | error 283.5550537109375 | self_attn.v_proj\n",
            "[3 / 12] |  time 0.33 | error 147548.0625 | self_attn.q_proj\n",
            "[3 / 12] |  time 0.33 | error 1.9796602725982666 | self_attn.out_proj\n",
            "[3 / 12] |  time 0.40 | error 1197.614501953125 | fc1\n",
            "[3 / 12] |  time 1.48 | error 9.484214782714844 | fc2\n",
            "quantizing activations ...\n",
            "[4 / 12] |  time 0.48 | error 373412.3125 | self_attn.k_proj\n",
            "[4 / 12] |  time 0.50 | error 351.716796875 | self_attn.v_proj\n",
            "[4 / 12] |  time 0.51 | error 287615.96875 | self_attn.q_proj\n",
            "[4 / 12] |  time 0.53 | error 3.198540687561035 | self_attn.out_proj\n",
            "[4 / 12] |  time 0.42 | error 751.5140991210938 | fc1\n",
            "[4 / 12] |  time 1.48 | error 7.594873428344727 | fc2\n",
            "quantizing activations ...\n",
            "[5 / 12] |  time 0.33 | error 208716.125 | self_attn.k_proj\n",
            "[5 / 12] |  time 0.33 | error 412.22698974609375 | self_attn.v_proj\n",
            "[5 / 12] |  time 0.34 | error 111473.0703125 | self_attn.q_proj\n",
            "[5 / 12] |  time 0.33 | error 4.4653730392456055 | self_attn.out_proj\n",
            "[5 / 12] |  time 0.40 | error 1099.4365234375 | fc1\n",
            "[5 / 12] |  time 1.51 | error 13.191829681396484 | fc2\n",
            "quantizing activations ...\n",
            "[6 / 12] |  time 0.55 | error 208503.59375 | self_attn.k_proj\n",
            "[6 / 12] |  time 0.50 | error 353.4970703125 | self_attn.v_proj\n",
            "[6 / 12] |  time 0.47 | error 97006.90625 | self_attn.q_proj\n",
            "[6 / 12] |  time 0.53 | error 5.336709022521973 | self_attn.out_proj\n",
            "[6 / 12] |  time 0.63 | error 853.1800537109375 | fc1\n",
            "[6 / 12] |  time 1.68 | error 19.74106788635254 | fc2\n",
            "quantizing activations ...\n",
            "[7 / 12] |  time 0.35 | error 158194.25 | self_attn.k_proj\n",
            "[7 / 12] |  time 0.32 | error 472.2510986328125 | self_attn.v_proj\n",
            "[7 / 12] |  time 0.33 | error 73440.546875 | self_attn.q_proj\n",
            "[7 / 12] |  time 0.36 | error 6.319820404052734 | self_attn.out_proj\n",
            "[7 / 12] |  time 0.41 | error 857.14697265625 | fc1\n",
            "[7 / 12] |  time 1.51 | error 28.573816299438477 | fc2\n",
            "quantizing activations ...\n",
            "[8 / 12] |  time 0.49 | error 137959.96875 | self_attn.k_proj\n",
            "[8 / 12] |  time 0.44 | error 772.9632568359375 | self_attn.v_proj\n",
            "[8 / 12] |  time 0.33 | error 41368.67578125 | self_attn.q_proj\n",
            "[8 / 12] |  time 0.32 | error 16.395751953125 | self_attn.out_proj\n",
            "[8 / 12] |  time 0.41 | error 1052.043212890625 | fc1\n",
            "[8 / 12] |  time 1.51 | error 29.984920501708984 | fc2\n",
            "quantizing activations ...\n",
            "[9 / 12] |  time 0.34 | error 108948.6796875 | self_attn.k_proj\n",
            "[9 / 12] |  time 0.32 | error 733.6072998046875 | self_attn.v_proj\n",
            "[9 / 12] |  time 0.34 | error 34592.8984375 | self_attn.q_proj\n",
            "[9 / 12] |  time 0.34 | error 20.590009689331055 | self_attn.out_proj\n",
            "[9 / 12] |  time 0.40 | error 1387.168212890625 | fc1\n",
            "[9 / 12] |  time 1.50 | error 40.712684631347656 | fc2\n",
            "quantizing activations ...\n",
            "[10 / 12] |  time 0.35 | error 59990.34375 | self_attn.k_proj\n",
            "[10 / 12] |  time 0.33 | error 786.9594116210938 | self_attn.v_proj\n",
            "[10 / 12] |  time 0.33 | error 17757.75390625 | self_attn.q_proj\n",
            "[10 / 12] |  time 0.35 | error 31.845346450805664 | self_attn.out_proj\n",
            "[10 / 12] |  time 0.41 | error 1687.5107421875 | fc1\n",
            "[10 / 12] |  time 1.52 | error 51.194786071777344 | fc2\n",
            "quantizing activations ...\n",
            "[11 / 12] |  time 0.36 | error 51073.1484375 | self_attn.k_proj\n",
            "[11 / 12] |  time 0.34 | error 939.4215698242188 | self_attn.v_proj\n",
            "[11 / 12] |  time 0.34 | error 22133.564453125 | self_attn.q_proj\n",
            "[11 / 12] |  time 0.35 | error 21.12183952331543 | self_attn.out_proj\n",
            "[11 / 12] |  time 0.41 | error 1942.523681640625 | fc1\n",
            "[11 / 12] |  time 1.59 | error 90.87786865234375 | fc2\n",
            "quantizing activations ...\n",
            "[12 / 12] |  time 0.35 | error 18142.16015625 | self_attn.k_proj\n",
            "[12 / 12] |  time 0.33 | error 1186.4888916015625 | self_attn.v_proj\n",
            "[12 / 12] |  time 0.34 | error 8031.18212890625 | self_attn.q_proj\n",
            "[12 / 12] |  time 0.34 | error 146.07342529296875 | self_attn.out_proj\n",
            "[12 / 12] |  time 0.40 | error 2785.61083984375 | fc1\n",
            "[12 / 12] |  time 1.71 | error 152.12686157226562 | fc2\n",
            "quantizing activations ...\n",
            "83 s\n",
            "/usr/local/lib/python3.10/dist-packages/datasets/load.py:1491: FutureWarning: The repository for ptb_text_only contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/ptb_text_only\n",
            "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
            "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
            "  warnings.warn(\n",
            "ptb\n",
            "Evaluating ...\n",
            "[0 / 12]\n",
            "[1 / 12]\n",
            "[2 / 12]\n",
            "[3 / 12]\n",
            "[4 / 12]\n",
            "[5 / 12]\n",
            "[6 / 12]\n",
            "[7 / 12]\n",
            "[8 / 12]\n",
            "[9 / 12]\n",
            "[10 / 12]\n",
            "[11 / 12]\n",
            "95 PPL\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Packing ...\n",
            "model.decoder.layers.0.self_attn.k_proj\n",
            "model.decoder.layers.0.self_attn.v_proj\n",
            "model.decoder.layers.0.self_attn.q_proj\n",
            "model.decoder.layers.0.self_attn.out_proj\n",
            "model.decoder.layers.0.fc1\n",
            "model.decoder.layers.0.fc2\n",
            "model.decoder.layers.1.self_attn.k_proj\n",
            "model.decoder.layers.1.self_attn.v_proj\n",
            "model.decoder.layers.1.self_attn.q_proj\n",
            "model.decoder.layers.1.self_attn.out_proj\n",
            "model.decoder.layers.1.fc1\n",
            "model.decoder.layers.1.fc2\n",
            "model.decoder.layers.2.self_attn.k_proj\n",
            "model.decoder.layers.2.self_attn.v_proj\n",
            "model.decoder.layers.2.self_attn.q_proj\n",
            "model.decoder.layers.2.self_attn.out_proj\n",
            "model.decoder.layers.2.fc1\n",
            "model.decoder.layers.2.fc2\n",
            "model.decoder.layers.3.self_attn.k_proj\n",
            "model.decoder.layers.3.self_attn.v_proj\n",
            "model.decoder.layers.3.self_attn.q_proj\n",
            "model.decoder.layers.3.self_attn.out_proj\n",
            "model.decoder.layers.3.fc1\n",
            "model.decoder.layers.3.fc2\n",
            "model.decoder.layers.4.self_attn.k_proj\n",
            "model.decoder.layers.4.self_attn.v_proj\n",
            "model.decoder.layers.4.self_attn.q_proj\n",
            "model.decoder.layers.4.self_attn.out_proj\n",
            "model.decoder.layers.4.fc1\n",
            "model.decoder.layers.4.fc2\n",
            "model.decoder.layers.5.self_attn.k_proj\n",
            "model.decoder.layers.5.self_attn.v_proj\n",
            "model.decoder.layers.5.self_attn.q_proj\n",
            "model.decoder.layers.5.self_attn.out_proj\n",
            "model.decoder.layers.5.fc1\n",
            "model.decoder.layers.5.fc2\n",
            "model.decoder.layers.6.self_attn.k_proj\n",
            "model.decoder.layers.6.self_attn.v_proj\n",
            "model.decoder.layers.6.self_attn.q_proj\n",
            "model.decoder.layers.6.self_attn.out_proj\n",
            "model.decoder.layers.6.fc1\n",
            "model.decoder.layers.6.fc2\n",
            "model.decoder.layers.7.self_attn.k_proj\n",
            "model.decoder.layers.7.self_attn.v_proj\n",
            "model.decoder.layers.7.self_attn.q_proj\n",
            "model.decoder.layers.7.self_attn.out_proj\n",
            "model.decoder.layers.7.fc1\n",
            "model.decoder.layers.7.fc2\n",
            "model.decoder.layers.8.self_attn.k_proj\n",
            "model.decoder.layers.8.self_attn.v_proj\n",
            "model.decoder.layers.8.self_attn.q_proj\n",
            "model.decoder.layers.8.self_attn.out_proj\n",
            "model.decoder.layers.8.fc1\n",
            "model.decoder.layers.8.fc2\n",
            "model.decoder.layers.9.self_attn.k_proj\n",
            "model.decoder.layers.9.self_attn.v_proj\n",
            "model.decoder.layers.9.self_attn.q_proj\n",
            "model.decoder.layers.9.self_attn.out_proj\n",
            "model.decoder.layers.9.fc1\n",
            "model.decoder.layers.9.fc2\n",
            "model.decoder.layers.10.self_attn.k_proj\n",
            "model.decoder.layers.10.self_attn.v_proj\n",
            "model.decoder.layers.10.self_attn.q_proj\n",
            "model.decoder.layers.10.self_attn.out_proj\n",
            "model.decoder.layers.10.fc1\n",
            "model.decoder.layers.10.fc2\n",
            "model.decoder.layers.11.self_attn.k_proj\n",
            "model.decoder.layers.11.self_attn.v_proj\n",
            "model.decoder.layers.11.self_attn.q_proj\n",
            "model.decoder.layers.11.self_attn.out_proj\n",
            "model.decoder.layers.11.fc1\n",
            "model.decoder.layers.11.fc2\n",
            "Done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python opt.py facebook/opt-125m c4 --load opt-125m-w4-a4-eig-n128.pt --benchmark 128 --check\n"
      ],
      "metadata": {
        "id": "mdo0C1P3pMD-",
        "outputId": "165a3139-6ded-4884-fcb8-981c1ebe4ddc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Loading model ...\n",
            "Done.\n",
            "Processed 256 sequences.\n",
            "Benchmarking ...\n",
            "0 0.8575999736785889\n",
            "1 0.02227330207824707\n",
            "2 0.014675617218017578\n",
            "3 0.015308618545532227\n",
            "4 0.014952421188354492\n",
            "5 0.014330625534057617\n",
            "6 0.014843225479125977\n",
            "7 0.014385700225830078\n",
            "8 0.01422262191772461\n",
            "9 0.016628026962280273\n",
            "10 0.014330387115478516\n",
            "11 0.014292001724243164\n",
            "12 0.01905202865600586\n",
            "13 0.014750003814697266\n",
            "14 0.014287948608398438\n",
            "15 0.013770103454589844\n",
            "16 0.016121864318847656\n",
            "17 0.013857603073120117\n",
            "18 0.014220714569091797\n",
            "19 0.014070510864257812\n",
            "20 0.014403581619262695\n",
            "21 0.014595508575439453\n",
            "22 0.013870477676391602\n",
            "23 0.014576435089111328\n",
            "24 0.013892889022827148\n",
            "25 0.01777482032775879\n",
            "26 0.01578807830810547\n",
            "27 0.016521215438842773\n",
            "28 0.01543879508972168\n",
            "29 0.015171289443969727\n",
            "30 0.013780355453491211\n",
            "31 0.014362573623657227\n",
            "32 0.014425516128540039\n",
            "33 0.014270305633544922\n",
            "34 0.014312744140625\n",
            "35 0.01428365707397461\n",
            "36 0.016237497329711914\n",
            "37 0.017919301986694336\n",
            "38 0.014968633651733398\n",
            "39 0.014111042022705078\n",
            "40 0.013934850692749023\n",
            "41 0.014513731002807617\n",
            "42 0.017802715301513672\n",
            "43 0.02629828453063965\n",
            "44 0.017447948455810547\n",
            "45 0.015961885452270508\n",
            "46 0.02057337760925293\n",
            "47 0.01899433135986328\n",
            "48 0.018262147903442383\n",
            "49 0.016021251678466797\n",
            "50 0.017311811447143555\n",
            "51 0.02225804328918457\n",
            "52 0.02531147003173828\n",
            "53 0.018559694290161133\n",
            "54 0.01988363265991211\n",
            "55 0.016516447067260742\n",
            "56 0.015676021575927734\n",
            "57 0.014962911605834961\n",
            "58 0.01555013656616211\n",
            "59 0.016851186752319336\n",
            "60 0.01988077163696289\n",
            "61 0.015398979187011719\n",
            "62 0.019680023193359375\n",
            "63 0.019915342330932617\n",
            "64 0.018644094467163086\n",
            "65 0.015827655792236328\n",
            "66 0.018170833587646484\n",
            "67 0.01601099967956543\n",
            "68 0.014430522918701172\n",
            "69 0.023328065872192383\n",
            "70 0.020862102508544922\n",
            "71 0.01764988899230957\n",
            "72 0.021660804748535156\n",
            "73 0.020316600799560547\n",
            "74 0.02090311050415039\n",
            "75 0.018750905990600586\n",
            "76 0.02025604248046875\n",
            "77 0.018766164779663086\n",
            "78 0.01858234405517578\n",
            "79 0.020488500595092773\n",
            "80 0.017780542373657227\n",
            "81 0.02059459686279297\n",
            "82 0.01756596565246582\n",
            "83 0.022046566009521484\n",
            "84 0.02996206283569336\n",
            "85 0.018975496292114258\n",
            "86 0.0172727108001709\n",
            "87 0.017658233642578125\n",
            "88 0.017315149307250977\n",
            "89 0.018077850341796875\n",
            "90 0.017002582550048828\n",
            "91 0.020129680633544922\n",
            "92 0.017065048217773438\n",
            "93 0.01715850830078125\n",
            "94 0.0172271728515625\n",
            "95 0.017011404037475586\n",
            "96 0.01638650894165039\n",
            "97 0.017650365829467773\n",
            "98 0.01738762855529785\n",
            "99 0.017128705978393555\n",
            "100 0.01689004898071289\n",
            "101 0.022756338119506836\n",
            "102 0.01698899269104004\n",
            "103 0.019690275192260742\n",
            "104 0.01743030548095703\n",
            "105 0.018384695053100586\n",
            "106 0.019258499145507812\n",
            "107 0.019828319549560547\n",
            "108 0.01905655860900879\n",
            "109 0.01942610740661621\n",
            "110 0.01887035369873047\n",
            "111 0.018804311752319336\n",
            "112 0.018570899963378906\n",
            "113 0.01827859878540039\n",
            "114 0.018608808517456055\n",
            "115 0.019280433654785156\n",
            "116 0.018694400787353516\n",
            "117 0.018494844436645508\n",
            "118 0.020139217376708984\n",
            "119 0.020877599716186523\n",
            "120 0.025154829025268555\n",
            "121 0.01733565330505371\n",
            "122 0.011553287506103516\n",
            "123 0.011293888092041016\n",
            "124 0.01191258430480957\n",
            "125 0.01139068603515625\n",
            "126 0.011316299438476562\n",
            "127 0.01164865493774414\n",
            "Median: 0.017292261123657227\n",
            "PPL: 21104.609375\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# ====================="
      ],
      "metadata": {
        "id": "Erqaa15nnQi7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# ABLATION"
      ],
      "metadata": {
        "id": "pOIPDZDFmwqG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## W4_A4_RTN_CLE\n"
      ],
      "metadata": {
        "id": "iozm7wJcm23m"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!python opt.py facebook/opt-125m c4 --wbits 4 --nsamples 128 --abits 4 --amethod rtn --cle 1 --save opt-125m-w4-a4-rtn-cle-n128.pt\n"
      ],
      "metadata": {
        "collapsed": true,
        "outputId": "05acdfaa-0643-4c4c-cc23-64d669c5ea06",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b_o_zlzDm23m"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processed 256 sequences.\n",
            "Starting ...\n",
            "Ready.\n",
            "iterating decoder layers...\n",
            "USING RTN ACTIVATION Q\n",
            "quantizing weights\n",
            "[1 / 12] |  time 0.49 | error 2778.73681640625 | self_attn.k_proj\n",
            "[1 / 12] |  time 0.31 | error 854.239501953125 | self_attn.v_proj\n",
            "[1 / 12] |  time 0.31 | error 5395.6357421875 | self_attn.q_proj\n",
            "[1 / 12] |  time 0.30 | error 3.012087821960449 | self_attn.out_proj\n",
            "[1 / 12] |  time 0.37 | error 123.57704162597656 | fc1\n",
            "[1 / 12] |  time 1.22 | error 4.977382659912109 | fc2\n",
            "quantizing activations ...\n",
            "[2 / 12] |  time 0.30 | error 1045.382568359375 | self_attn.k_proj\n",
            "[2 / 12] |  time 0.30 | error 481.5646057128906 | self_attn.v_proj\n",
            "[2 / 12] |  time 0.44 | error 2823.690185546875 | self_attn.q_proj\n",
            "[2 / 12] |  time 0.40 | error 2.4591166973114014 | self_attn.out_proj\n",
            "[2 / 12] |  time 0.47 | error 1350.5838623046875 | fc1\n",
            "[2 / 12] |  time 1.61 | error 7.787049770355225 | fc2\n",
            "quantizing activations ...\n",
            "[3 / 12] |  time 0.31 | error 1279.6695556640625 | self_attn.k_proj\n",
            "[3 / 12] |  time 0.30 | error 1121.94580078125 | self_attn.v_proj\n",
            "[3 / 12] |  time 0.32 | error 2129.7822265625 | self_attn.q_proj\n",
            "[3 / 12] |  time 0.31 | error 16.736919403076172 | self_attn.out_proj\n",
            "[3 / 12] |  time 0.36 | error 222.72300720214844 | fc1\n",
            "[3 / 12] |  time 1.20 | error 1.9769057035446167 | fc2\n",
            "quantizing activations ...\n",
            "[4 / 12] |  time 0.50 | error 1299.2197265625 | self_attn.k_proj\n",
            "[4 / 12] |  time 0.49 | error 193.87071228027344 | self_attn.v_proj\n",
            "[4 / 12] |  time 0.51 | error 1462.826904296875 | self_attn.q_proj\n",
            "[4 / 12] |  time 0.32 | error 7.303255081176758 | self_attn.out_proj\n",
            "[4 / 12] |  time 0.38 | error 132.62954711914062 | fc1\n",
            "[4 / 12] |  time 1.24 | error 1.1826558113098145 | fc2\n",
            "quantizing activations ...\n",
            "[5 / 12] |  time 0.30 | error 1153.605224609375 | self_attn.k_proj\n",
            "[5 / 12] |  time 0.32 | error 214.75665283203125 | self_attn.v_proj\n",
            "[5 / 12] |  time 0.33 | error 995.4599609375 | self_attn.q_proj\n",
            "[5 / 12] |  time 0.32 | error 24.245725631713867 | self_attn.out_proj\n",
            "[5 / 12] |  time 0.37 | error 96.25282287597656 | fc1\n",
            "[5 / 12] |  time 1.23 | error 0.8286391496658325 | fc2\n",
            "quantizing activations ...\n",
            "[6 / 12] |  time 0.33 | error 317.46612548828125 | self_attn.k_proj\n",
            "[6 / 12] |  time 0.31 | error 352.2181396484375 | self_attn.v_proj\n",
            "[6 / 12] |  time 0.32 | error 273.205322265625 | self_attn.q_proj\n",
            "[6 / 12] |  time 0.34 | error 127.1017074584961 | self_attn.out_proj\n",
            "[6 / 12] |  time 0.38 | error 45.30931091308594 | fc1\n",
            "[6 / 12] |  time 1.21 | error 0.9335471391677856 | fc2\n",
            "quantizing activations ...\n",
            "[7 / 12] |  time 0.32 | error 164.976806640625 | self_attn.k_proj\n",
            "[7 / 12] |  time 0.30 | error 206.06068420410156 | self_attn.v_proj\n",
            "[7 / 12] |  time 0.30 | error 228.48573303222656 | self_attn.q_proj\n",
            "[7 / 12] |  time 0.33 | error 109.86384582519531 | self_attn.out_proj\n",
            "[7 / 12] |  time 0.37 | error 65.78901672363281 | fc1\n",
            "[7 / 12] |  time 1.29 | error 1.1018732786178589 | fc2\n",
            "quantizing activations ...\n",
            "[8 / 12] |  time 0.31 | error 106.012451171875 | self_attn.k_proj\n",
            "[8 / 12] |  time 0.33 | error 232.07461547851562 | self_attn.v_proj\n",
            "[8 / 12] |  time 0.31 | error 121.40861511230469 | self_attn.q_proj\n",
            "[8 / 12] |  time 0.30 | error 265.0511474609375 | self_attn.out_proj\n",
            "[8 / 12] |  time 0.37 | error 65.89734649658203 | fc1\n",
            "[8 / 12] |  time 1.22 | error 0.8058255910873413 | fc2\n",
            "quantizing activations ...\n",
            "[9 / 12] |  time 0.31 | error 184.8526153564453 | self_attn.k_proj\n",
            "[9 / 12] |  time 0.30 | error 117.49838256835938 | self_attn.v_proj\n",
            "[9 / 12] |  time 0.31 | error 207.40615844726562 | self_attn.q_proj\n",
            "[9 / 12] |  time 0.32 | error 57.925514221191406 | self_attn.out_proj\n",
            "[9 / 12] |  time 0.43 | error 93.919189453125 | fc1\n",
            "[9 / 12] |  time 1.69 | error 2.323619842529297 | fc2\n",
            "quantizing activations ...\n",
            "[10 / 12] |  time 0.30 | error 198.96754455566406 | self_attn.k_proj\n",
            "[10 / 12] |  time 0.33 | error 137.94680786132812 | self_attn.v_proj\n",
            "[10 / 12] |  time 0.32 | error 204.16668701171875 | self_attn.q_proj\n",
            "[10 / 12] |  time 0.31 | error 70.43637084960938 | self_attn.out_proj\n",
            "[10 / 12] |  time 0.38 | error 38.339908599853516 | fc1\n",
            "[10 / 12] |  time 1.21 | error 3.6083977222442627 | fc2\n",
            "quantizing activations ...\n",
            "[11 / 12] |  time 0.40 | error 334.443603515625 | self_attn.k_proj\n",
            "[11 / 12] |  time 0.42 | error 130.72857666015625 | self_attn.v_proj\n",
            "[11 / 12] |  time 0.48 | error 189.74407958984375 | self_attn.q_proj\n",
            "[11 / 12] |  time 0.48 | error 88.3185043334961 | self_attn.out_proj\n",
            "[11 / 12] |  time 0.50 | error 120.15911865234375 | fc1\n",
            "[11 / 12] |  time 1.22 | error 3.319108724594116 | fc2\n",
            "quantizing activations ...\n",
            "[12 / 12] |  time 0.32 | error 440.9690246582031 | self_attn.k_proj\n",
            "[12 / 12] |  time 0.30 | error 66.60034942626953 | self_attn.v_proj\n",
            "[12 / 12] |  time 0.32 | error 613.1421508789062 | self_attn.q_proj\n",
            "[12 / 12] |  time 0.34 | error 10.16918659210205 | self_attn.out_proj\n",
            "[12 / 12] |  time 0.37 | error 270.9554443359375 | fc1\n",
            "[12 / 12] |  time 1.22 | error 11.168766021728516 | fc2\n",
            "quantizing activations ...\n",
            "87 s\n",
            "/usr/local/lib/python3.10/dist-packages/datasets/load.py:1491: FutureWarning: The repository for ptb_text_only contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/ptb_text_only\n",
            "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
            "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
            "  warnings.warn(\n",
            "ptb\n",
            "Evaluating ...\n",
            "[0 / 12]\n",
            "[1 / 12]\n",
            "[2 / 12]\n",
            "[3 / 12]\n",
            "[4 / 12]\n",
            "[5 / 12]\n",
            "[6 / 12]\n",
            "[7 / 12]\n",
            "[8 / 12]\n",
            "[9 / 12]\n",
            "[10 / 12]\n",
            "[11 / 12]\n",
            "10077 PPL\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Packing ...\n",
            "model.decoder.layers.0.self_attn.k_proj\n",
            "model.decoder.layers.0.self_attn.v_proj\n",
            "model.decoder.layers.0.self_attn.q_proj\n",
            "model.decoder.layers.0.self_attn.out_proj\n",
            "model.decoder.layers.0.fc1\n",
            "model.decoder.layers.0.fc2\n",
            "model.decoder.layers.1.self_attn.k_proj\n",
            "model.decoder.layers.1.self_attn.v_proj\n",
            "model.decoder.layers.1.self_attn.q_proj\n",
            "model.decoder.layers.1.self_attn.out_proj\n",
            "model.decoder.layers.1.fc1\n",
            "model.decoder.layers.1.fc2\n",
            "model.decoder.layers.2.self_attn.k_proj\n",
            "model.decoder.layers.2.self_attn.v_proj\n",
            "model.decoder.layers.2.self_attn.q_proj\n",
            "model.decoder.layers.2.self_attn.out_proj\n",
            "model.decoder.layers.2.fc1\n",
            "model.decoder.layers.2.fc2\n",
            "model.decoder.layers.3.self_attn.k_proj\n",
            "model.decoder.layers.3.self_attn.v_proj\n",
            "model.decoder.layers.3.self_attn.q_proj\n",
            "model.decoder.layers.3.self_attn.out_proj\n",
            "model.decoder.layers.3.fc1\n",
            "model.decoder.layers.3.fc2\n",
            "model.decoder.layers.4.self_attn.k_proj\n",
            "model.decoder.layers.4.self_attn.v_proj\n",
            "model.decoder.layers.4.self_attn.q_proj\n",
            "model.decoder.layers.4.self_attn.out_proj\n",
            "model.decoder.layers.4.fc1\n",
            "model.decoder.layers.4.fc2\n",
            "model.decoder.layers.5.self_attn.k_proj\n",
            "model.decoder.layers.5.self_attn.v_proj\n",
            "model.decoder.layers.5.self_attn.q_proj\n",
            "model.decoder.layers.5.self_attn.out_proj\n",
            "model.decoder.layers.5.fc1\n",
            "model.decoder.layers.5.fc2\n",
            "model.decoder.layers.6.self_attn.k_proj\n",
            "model.decoder.layers.6.self_attn.v_proj\n",
            "model.decoder.layers.6.self_attn.q_proj\n",
            "model.decoder.layers.6.self_attn.out_proj\n",
            "model.decoder.layers.6.fc1\n",
            "model.decoder.layers.6.fc2\n",
            "model.decoder.layers.7.self_attn.k_proj\n",
            "model.decoder.layers.7.self_attn.v_proj\n",
            "model.decoder.layers.7.self_attn.q_proj\n",
            "model.decoder.layers.7.self_attn.out_proj\n",
            "model.decoder.layers.7.fc1\n",
            "model.decoder.layers.7.fc2\n",
            "model.decoder.layers.8.self_attn.k_proj\n",
            "model.decoder.layers.8.self_attn.v_proj\n",
            "model.decoder.layers.8.self_attn.q_proj\n",
            "model.decoder.layers.8.self_attn.out_proj\n",
            "model.decoder.layers.8.fc1\n",
            "model.decoder.layers.8.fc2\n",
            "model.decoder.layers.9.self_attn.k_proj\n",
            "model.decoder.layers.9.self_attn.v_proj\n",
            "model.decoder.layers.9.self_attn.q_proj\n",
            "model.decoder.layers.9.self_attn.out_proj\n",
            "model.decoder.layers.9.fc1\n",
            "model.decoder.layers.9.fc2\n",
            "model.decoder.layers.10.self_attn.k_proj\n",
            "model.decoder.layers.10.self_attn.v_proj\n",
            "model.decoder.layers.10.self_attn.q_proj\n",
            "model.decoder.layers.10.self_attn.out_proj\n",
            "model.decoder.layers.10.fc1\n",
            "model.decoder.layers.10.fc2\n",
            "model.decoder.layers.11.self_attn.k_proj\n",
            "model.decoder.layers.11.self_attn.v_proj\n",
            "model.decoder.layers.11.self_attn.q_proj\n",
            "model.decoder.layers.11.self_attn.out_proj\n",
            "model.decoder.layers.11.fc1\n",
            "model.decoder.layers.11.fc2\n",
            "Done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python opt.py facebook/opt-125m c4 --load opt-125m-w4-a4-rtn-cle-n128.pt --benchmark 128 --check\n"
      ],
      "metadata": {
        "collapsed": true,
        "outputId": "f44f832c-a587-4fdf-d205-813a2f89fb8d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JYMaScz_m23n"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Loading model ...\n",
            "Done.\n",
            "Processed 256 sequences.\n",
            "Benchmarking ...\n",
            "0 0.7383677959442139\n",
            "1 0.019917011260986328\n",
            "2 0.01295924186706543\n",
            "3 0.012560606002807617\n",
            "4 0.013163328170776367\n",
            "5 0.012561321258544922\n",
            "6 0.012495756149291992\n",
            "7 0.013252019882202148\n",
            "8 0.01265263557434082\n",
            "9 0.01248478889465332\n",
            "10 0.01121377944946289\n",
            "11 0.01110982894897461\n",
            "12 0.011074304580688477\n",
            "13 0.011363506317138672\n",
            "14 0.01114201545715332\n",
            "15 0.011152982711791992\n",
            "16 0.01121377944946289\n",
            "17 0.011481761932373047\n",
            "18 0.011129140853881836\n",
            "19 0.012408733367919922\n",
            "20 0.013751983642578125\n",
            "21 0.013568401336669922\n",
            "22 0.011800050735473633\n",
            "23 0.012503385543823242\n",
            "24 0.011638879776000977\n",
            "25 0.011299848556518555\n",
            "26 0.01121664047241211\n",
            "27 0.011156320571899414\n",
            "28 0.012052774429321289\n",
            "29 0.011528968811035156\n",
            "30 0.016844749450683594\n",
            "31 0.015668630599975586\n",
            "32 0.011677265167236328\n",
            "33 0.01184844970703125\n",
            "34 0.01143956184387207\n",
            "35 0.011152029037475586\n",
            "36 0.011958122253417969\n",
            "37 0.0114898681640625\n",
            "38 0.011371850967407227\n",
            "39 0.011615276336669922\n",
            "40 0.011809110641479492\n",
            "41 0.011286497116088867\n",
            "42 0.011057615280151367\n",
            "43 0.011035919189453125\n",
            "44 0.010864734649658203\n",
            "45 0.013948917388916016\n",
            "46 0.011140108108520508\n",
            "47 0.01251220703125\n",
            "48 0.0130615234375\n",
            "49 0.011510610580444336\n",
            "50 0.01151895523071289\n",
            "51 0.011842727661132812\n",
            "52 0.011626243591308594\n",
            "53 0.011690616607666016\n",
            "54 0.01096034049987793\n",
            "55 0.011911153793334961\n",
            "56 0.011494874954223633\n",
            "57 0.010954856872558594\n",
            "58 0.011255979537963867\n",
            "59 0.012511491775512695\n",
            "60 0.012098312377929688\n",
            "61 0.011517524719238281\n",
            "62 0.011409997940063477\n",
            "63 0.011167526245117188\n",
            "64 0.011309385299682617\n",
            "65 0.01164388656616211\n",
            "66 0.011040210723876953\n",
            "67 0.011173486709594727\n",
            "68 0.016099214553833008\n",
            "69 0.014504671096801758\n",
            "70 0.014270782470703125\n",
            "71 0.012126445770263672\n",
            "72 0.01462554931640625\n",
            "73 0.014136075973510742\n",
            "74 0.01228475570678711\n",
            "75 0.012397050857543945\n",
            "76 0.011298656463623047\n",
            "77 0.01253509521484375\n",
            "78 0.012154817581176758\n",
            "79 0.012687206268310547\n",
            "80 0.011826276779174805\n",
            "81 0.012632608413696289\n",
            "82 0.012148141860961914\n",
            "83 0.011210918426513672\n",
            "84 0.011436700820922852\n",
            "85 0.01157069206237793\n",
            "86 0.011082887649536133\n",
            "87 0.01091313362121582\n",
            "88 0.011055231094360352\n",
            "89 0.011380910873413086\n",
            "90 0.011053323745727539\n",
            "91 0.010982751846313477\n",
            "92 0.01115107536315918\n",
            "93 0.011249065399169922\n",
            "94 0.01412510871887207\n",
            "95 0.011512279510498047\n",
            "96 0.011221647262573242\n",
            "97 0.011113405227661133\n",
            "98 0.013965845108032227\n",
            "99 0.011552810668945312\n",
            "100 0.011042118072509766\n",
            "101 0.012749195098876953\n",
            "102 0.01154327392578125\n",
            "103 0.011811971664428711\n",
            "104 0.012283563613891602\n",
            "105 0.011033058166503906\n",
            "106 0.012469768524169922\n",
            "107 0.011905670166015625\n",
            "108 0.011075496673583984\n",
            "109 0.011083602905273438\n",
            "110 0.011080265045166016\n",
            "111 0.011333703994750977\n",
            "112 0.011108875274658203\n",
            "113 0.011088848114013672\n",
            "114 0.014099836349487305\n",
            "115 0.018294811248779297\n",
            "116 0.01119852066040039\n",
            "117 0.011411905288696289\n",
            "118 0.011501550674438477\n",
            "119 0.014448165893554688\n",
            "120 0.011139154434204102\n",
            "121 0.011807680130004883\n",
            "122 0.011507511138916016\n",
            "123 0.014525651931762695\n",
            "124 0.011210203170776367\n",
            "125 0.011335372924804688\n",
            "126 0.011211633682250977\n",
            "127 0.011700630187988281\n",
            "Median: 0.011536121368408203\n",
            "PPL: 14043.1435546875\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## W4_A4_RTN_EIG\n"
      ],
      "metadata": {
        "id": "h7bHFPgSm3N_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!python opt.py facebook/opt-125m c4 --wbits 4 --nsamples 128 --abits 4 --amethod rtn --eigenvalues 1 --save opt-125m-w4-a4-rtn-eig-n128.pt\n"
      ],
      "metadata": {
        "collapsed": true,
        "outputId": "1d64e6d3-a2b0-47b3-d583-9cc8dc52393c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sirl5Kozm3OA"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processed 256 sequences.\n",
            "Starting ...\n",
            "Ready.\n",
            "iterating decoder layers...\n",
            "USING RTN ACTIVATION Q\n",
            "quantizing weights\n",
            "[1 / 12] |  time 0.52 | error 25699.59375 | self_attn.k_proj\n",
            "[1 / 12] |  time 0.35 | error 196.69696044921875 | self_attn.v_proj\n",
            "[1 / 12] |  time 0.33 | error 18219.3515625 | self_attn.q_proj\n",
            "[1 / 12] |  time 0.33 | error 3.203965663909912 | self_attn.out_proj\n",
            "[1 / 12] |  time 0.42 | error 252.86203002929688 | fc1\n",
            "[1 / 12] |  time 1.53 | error 431.032470703125 | fc2\n",
            "quantizing activations ...\n",
            "[2 / 12] |  time 0.49 | error 25806.57421875 | self_attn.k_proj\n",
            "[2 / 12] |  time 0.51 | error 120.58409881591797 | self_attn.v_proj\n",
            "[2 / 12] |  time 0.52 | error 10014.92578125 | self_attn.q_proj\n",
            "[2 / 12] |  time 0.38 | error 4.692255020141602 | self_attn.out_proj\n",
            "[2 / 12] |  time 0.40 | error 1314.8114013671875 | fc1\n",
            "[2 / 12] |  time 1.48 | error 10.216139793395996 | fc2\n",
            "quantizing activations ...\n",
            "[3 / 12] |  time 0.50 | error 131956.625 | self_attn.k_proj\n",
            "[3 / 12] |  time 0.34 | error 283.5550537109375 | self_attn.v_proj\n",
            "[3 / 12] |  time 0.35 | error 147548.0625 | self_attn.q_proj\n",
            "[3 / 12] |  time 0.34 | error 1.9796602725982666 | self_attn.out_proj\n",
            "[3 / 12] |  time 0.41 | error 1197.614501953125 | fc1\n",
            "[3 / 12] |  time 1.48 | error 9.484214782714844 | fc2\n",
            "quantizing activations ...\n",
            "[4 / 12] |  time 0.37 | error 373412.3125 | self_attn.k_proj\n",
            "[4 / 12] |  time 0.32 | error 351.716796875 | self_attn.v_proj\n",
            "[4 / 12] |  time 0.33 | error 287615.96875 | self_attn.q_proj\n",
            "[4 / 12] |  time 0.33 | error 3.198540687561035 | self_attn.out_proj\n",
            "[4 / 12] |  time 0.42 | error 751.5140991210938 | fc1\n",
            "[4 / 12] |  time 1.46 | error 7.594873428344727 | fc2\n",
            "quantizing activations ...\n",
            "[5 / 12] |  time 0.34 | error 208716.125 | self_attn.k_proj\n",
            "[5 / 12] |  time 0.32 | error 412.22698974609375 | self_attn.v_proj\n",
            "[5 / 12] |  time 0.33 | error 111473.0703125 | self_attn.q_proj\n",
            "[5 / 12] |  time 0.34 | error 4.4653730392456055 | self_attn.out_proj\n",
            "[5 / 12] |  time 0.54 | error 1099.4365234375 | fc1\n",
            "[5 / 12] |  time 1.99 | error 13.191829681396484 | fc2\n",
            "quantizing activations ...\n",
            "[6 / 12] |  time 0.34 | error 208503.59375 | self_attn.k_proj\n",
            "[6 / 12] |  time 0.33 | error 353.4970703125 | self_attn.v_proj\n",
            "[6 / 12] |  time 0.33 | error 97006.90625 | self_attn.q_proj\n",
            "[6 / 12] |  time 0.33 | error 5.336709022521973 | self_attn.out_proj\n",
            "[6 / 12] |  time 0.40 | error 853.1800537109375 | fc1\n",
            "[6 / 12] |  time 1.50 | error 19.74106788635254 | fc2\n",
            "quantizing activations ...\n",
            "[7 / 12] |  time 0.50 | error 158194.25 | self_attn.k_proj\n",
            "[7 / 12] |  time 0.46 | error 472.2510986328125 | self_attn.v_proj\n",
            "[7 / 12] |  time 0.42 | error 73440.546875 | self_attn.q_proj\n",
            "[7 / 12] |  time 0.32 | error 6.319820404052734 | self_attn.out_proj\n",
            "[7 / 12] |  time 0.40 | error 857.14697265625 | fc1\n",
            "[7 / 12] |  time 1.50 | error 28.573816299438477 | fc2\n",
            "quantizing activations ...\n",
            "[8 / 12] |  time 0.34 | error 137959.96875 | self_attn.k_proj\n",
            "[8 / 12] |  time 0.34 | error 772.9632568359375 | self_attn.v_proj\n",
            "[8 / 12] |  time 0.33 | error 41368.67578125 | self_attn.q_proj\n",
            "[8 / 12] |  time 0.35 | error 16.395751953125 | self_attn.out_proj\n",
            "[8 / 12] |  time 0.44 | error 1052.043212890625 | fc1\n",
            "[8 / 12] |  time 1.49 | error 29.984920501708984 | fc2\n",
            "quantizing activations ...\n",
            "[9 / 12] |  time 0.34 | error 108948.6796875 | self_attn.k_proj\n",
            "[9 / 12] |  time 0.32 | error 733.6072998046875 | self_attn.v_proj\n",
            "[9 / 12] |  time 0.32 | error 34592.8984375 | self_attn.q_proj\n",
            "[9 / 12] |  time 0.33 | error 20.590009689331055 | self_attn.out_proj\n",
            "[9 / 12] |  time 0.42 | error 1387.168212890625 | fc1\n",
            "[9 / 12] |  time 1.52 | error 40.712684631347656 | fc2\n",
            "quantizing activations ...\n",
            "[10 / 12] |  time 0.34 | error 59990.34375 | self_attn.k_proj\n",
            "[10 / 12] |  time 0.34 | error 786.9594116210938 | self_attn.v_proj\n",
            "[10 / 12] |  time 0.34 | error 17757.75390625 | self_attn.q_proj\n",
            "[10 / 12] |  time 0.35 | error 31.845346450805664 | self_attn.out_proj\n",
            "[10 / 12] |  time 0.52 | error 1687.5107421875 | fc1\n",
            "[10 / 12] |  time 2.07 | error 51.194786071777344 | fc2\n",
            "quantizing activations ...\n",
            "[11 / 12] |  time 0.33 | error 51073.1484375 | self_attn.k_proj\n",
            "[11 / 12] |  time 0.33 | error 939.4215698242188 | self_attn.v_proj\n",
            "[11 / 12] |  time 0.33 | error 22133.564453125 | self_attn.q_proj\n",
            "[11 / 12] |  time 0.32 | error 21.12183952331543 | self_attn.out_proj\n",
            "[11 / 12] |  time 0.40 | error 1942.523681640625 | fc1\n",
            "[11 / 12] |  time 1.48 | error 90.87786865234375 | fc2\n",
            "quantizing activations ...\n",
            "[12 / 12] |  time 0.51 | error 18142.16015625 | self_attn.k_proj\n",
            "[12 / 12] |  time 0.47 | error 1186.4888916015625 | self_attn.v_proj\n",
            "[12 / 12] |  time 0.36 | error 8031.18212890625 | self_attn.q_proj\n",
            "[12 / 12] |  time 0.37 | error 146.07342529296875 | self_attn.out_proj\n",
            "[12 / 12] |  time 0.39 | error 2785.61083984375 | fc1\n",
            "[12 / 12] |  time 1.52 | error 152.12686157226562 | fc2\n",
            "quantizing activations ...\n",
            "92 s\n",
            "/usr/local/lib/python3.10/dist-packages/datasets/load.py:1491: FutureWarning: The repository for ptb_text_only contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/ptb_text_only\n",
            "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
            "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
            "  warnings.warn(\n",
            "ptb\n",
            "Evaluating ...\n",
            "[0 / 12]\n",
            "[1 / 12]\n",
            "[2 / 12]\n",
            "[3 / 12]\n",
            "[4 / 12]\n",
            "[5 / 12]\n",
            "[6 / 12]\n",
            "[7 / 12]\n",
            "[8 / 12]\n",
            "[9 / 12]\n",
            "[10 / 12]\n",
            "[11 / 12]\n",
            "95 PPL\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Packing ...\n",
            "model.decoder.layers.0.self_attn.k_proj\n",
            "model.decoder.layers.0.self_attn.v_proj\n",
            "model.decoder.layers.0.self_attn.q_proj\n",
            "model.decoder.layers.0.self_attn.out_proj\n",
            "model.decoder.layers.0.fc1\n",
            "model.decoder.layers.0.fc2\n",
            "model.decoder.layers.1.self_attn.k_proj\n",
            "model.decoder.layers.1.self_attn.v_proj\n",
            "model.decoder.layers.1.self_attn.q_proj\n",
            "model.decoder.layers.1.self_attn.out_proj\n",
            "model.decoder.layers.1.fc1\n",
            "model.decoder.layers.1.fc2\n",
            "model.decoder.layers.2.self_attn.k_proj\n",
            "model.decoder.layers.2.self_attn.v_proj\n",
            "model.decoder.layers.2.self_attn.q_proj\n",
            "model.decoder.layers.2.self_attn.out_proj\n",
            "model.decoder.layers.2.fc1\n",
            "model.decoder.layers.2.fc2\n",
            "model.decoder.layers.3.self_attn.k_proj\n",
            "model.decoder.layers.3.self_attn.v_proj\n",
            "model.decoder.layers.3.self_attn.q_proj\n",
            "model.decoder.layers.3.self_attn.out_proj\n",
            "model.decoder.layers.3.fc1\n",
            "model.decoder.layers.3.fc2\n",
            "model.decoder.layers.4.self_attn.k_proj\n",
            "model.decoder.layers.4.self_attn.v_proj\n",
            "model.decoder.layers.4.self_attn.q_proj\n",
            "model.decoder.layers.4.self_attn.out_proj\n",
            "model.decoder.layers.4.fc1\n",
            "model.decoder.layers.4.fc2\n",
            "model.decoder.layers.5.self_attn.k_proj\n",
            "model.decoder.layers.5.self_attn.v_proj\n",
            "model.decoder.layers.5.self_attn.q_proj\n",
            "model.decoder.layers.5.self_attn.out_proj\n",
            "model.decoder.layers.5.fc1\n",
            "model.decoder.layers.5.fc2\n",
            "model.decoder.layers.6.self_attn.k_proj\n",
            "model.decoder.layers.6.self_attn.v_proj\n",
            "model.decoder.layers.6.self_attn.q_proj\n",
            "model.decoder.layers.6.self_attn.out_proj\n",
            "model.decoder.layers.6.fc1\n",
            "model.decoder.layers.6.fc2\n",
            "model.decoder.layers.7.self_attn.k_proj\n",
            "model.decoder.layers.7.self_attn.v_proj\n",
            "model.decoder.layers.7.self_attn.q_proj\n",
            "model.decoder.layers.7.self_attn.out_proj\n",
            "model.decoder.layers.7.fc1\n",
            "model.decoder.layers.7.fc2\n",
            "model.decoder.layers.8.self_attn.k_proj\n",
            "model.decoder.layers.8.self_attn.v_proj\n",
            "model.decoder.layers.8.self_attn.q_proj\n",
            "model.decoder.layers.8.self_attn.out_proj\n",
            "model.decoder.layers.8.fc1\n",
            "model.decoder.layers.8.fc2\n",
            "model.decoder.layers.9.self_attn.k_proj\n",
            "model.decoder.layers.9.self_attn.v_proj\n",
            "model.decoder.layers.9.self_attn.q_proj\n",
            "model.decoder.layers.9.self_attn.out_proj\n",
            "model.decoder.layers.9.fc1\n",
            "model.decoder.layers.9.fc2\n",
            "model.decoder.layers.10.self_attn.k_proj\n",
            "model.decoder.layers.10.self_attn.v_proj\n",
            "model.decoder.layers.10.self_attn.q_proj\n",
            "model.decoder.layers.10.self_attn.out_proj\n",
            "model.decoder.layers.10.fc1\n",
            "model.decoder.layers.10.fc2\n",
            "model.decoder.layers.11.self_attn.k_proj\n",
            "model.decoder.layers.11.self_attn.v_proj\n",
            "model.decoder.layers.11.self_attn.q_proj\n",
            "model.decoder.layers.11.self_attn.out_proj\n",
            "model.decoder.layers.11.fc1\n",
            "model.decoder.layers.11.fc2\n",
            "Done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python opt.py facebook/opt-125m c4 --load opt-125m-w4-a4-rtn-eig-n128.pt --benchmark 128 --check\n"
      ],
      "metadata": {
        "collapsed": true,
        "outputId": "a099506c-4e11-429c-84fb-27b2cc17d173",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wr06ElKlm3OA"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Loading model ...\n",
            "Done.\n",
            "Processed 256 sequences.\n",
            "Benchmarking ...\n",
            "0 0.7097940444946289\n",
            "1 0.016930103302001953\n",
            "2 0.011673450469970703\n",
            "3 0.011388301849365234\n",
            "4 0.011173725128173828\n",
            "5 0.011042594909667969\n",
            "6 0.010908126831054688\n",
            "7 0.011228799819946289\n",
            "8 0.011492013931274414\n",
            "9 0.011603832244873047\n",
            "10 0.011668682098388672\n",
            "11 0.01119852066040039\n",
            "12 0.011423587799072266\n",
            "13 0.011492490768432617\n",
            "14 0.011321067810058594\n",
            "15 0.011325359344482422\n",
            "16 0.011456012725830078\n",
            "17 0.01146554946899414\n",
            "18 0.012686014175415039\n",
            "19 0.013614177703857422\n",
            "20 0.013229846954345703\n",
            "21 0.011415243148803711\n",
            "22 0.011034250259399414\n",
            "23 0.01193857192993164\n",
            "24 0.011536836624145508\n",
            "25 0.011113643646240234\n",
            "26 0.010926008224487305\n",
            "27 0.01168370246887207\n",
            "28 0.0112762451171875\n",
            "29 0.011482477188110352\n",
            "30 0.01170969009399414\n",
            "31 0.011205196380615234\n",
            "32 0.011546134948730469\n",
            "33 0.011258602142333984\n",
            "34 0.011313676834106445\n",
            "35 0.011487960815429688\n",
            "36 0.011318683624267578\n",
            "37 0.011342525482177734\n",
            "38 0.011452198028564453\n",
            "39 0.011290788650512695\n",
            "40 0.012102603912353516\n",
            "41 0.011222600936889648\n",
            "42 0.011253833770751953\n",
            "43 0.012652397155761719\n",
            "44 0.018218278884887695\n",
            "45 0.011268138885498047\n",
            "46 0.011977910995483398\n",
            "47 0.011581897735595703\n",
            "48 0.011190414428710938\n",
            "49 0.01140284538269043\n",
            "50 0.0118408203125\n",
            "51 0.011392831802368164\n",
            "52 0.011836767196655273\n",
            "53 0.01121664047241211\n",
            "54 0.013035774230957031\n",
            "55 0.01133418083190918\n",
            "56 0.011174440383911133\n",
            "57 0.011447668075561523\n",
            "58 0.011402130126953125\n",
            "59 0.011349916458129883\n",
            "60 0.011472225189208984\n",
            "61 0.014061450958251953\n",
            "62 0.019808292388916016\n",
            "63 0.011307954788208008\n",
            "64 0.011496782302856445\n",
            "65 0.011323213577270508\n",
            "66 0.011200189590454102\n",
            "67 0.011339187622070312\n",
            "68 0.014101982116699219\n",
            "69 0.012896299362182617\n",
            "70 0.012002229690551758\n",
            "71 0.011253118515014648\n",
            "72 0.011116266250610352\n",
            "73 0.011843681335449219\n",
            "74 0.011031866073608398\n",
            "75 0.01145482063293457\n",
            "76 0.011647939682006836\n",
            "77 0.011109590530395508\n",
            "78 0.011245965957641602\n",
            "79 0.01113581657409668\n",
            "80 0.010996580123901367\n",
            "81 0.01093435287475586\n",
            "82 0.011105060577392578\n",
            "83 0.011656522750854492\n",
            "84 0.011370658874511719\n",
            "85 0.014192581176757812\n",
            "86 0.018486738204956055\n",
            "87 0.01700139045715332\n",
            "88 0.018267154693603516\n",
            "89 0.01780867576599121\n",
            "90 0.017151832580566406\n",
            "91 0.023748159408569336\n",
            "92 0.014885663986206055\n",
            "93 0.017032623291015625\n",
            "94 0.017446279525756836\n",
            "95 0.015042781829833984\n",
            "96 0.014125347137451172\n",
            "97 0.018280744552612305\n",
            "98 0.02208256721496582\n",
            "99 0.018304109573364258\n",
            "100 0.015367269515991211\n",
            "101 0.015242576599121094\n",
            "102 0.019176244735717773\n",
            "103 0.015407323837280273\n",
            "104 0.014119863510131836\n",
            "105 0.016030550003051758\n",
            "106 0.014710426330566406\n",
            "107 0.015351295471191406\n",
            "108 0.025215625762939453\n",
            "109 0.015180349349975586\n",
            "110 0.015793323516845703\n",
            "111 0.01476287841796875\n",
            "112 0.014629364013671875\n",
            "113 0.014797210693359375\n",
            "114 0.014628410339355469\n",
            "115 0.015646696090698242\n",
            "116 0.014969110488891602\n",
            "117 0.014519929885864258\n",
            "118 0.014524459838867188\n",
            "119 0.01471257209777832\n",
            "120 0.014500617980957031\n",
            "121 0.019501447677612305\n",
            "122 0.016422033309936523\n",
            "123 0.015852689743041992\n",
            "124 0.014795303344726562\n",
            "125 0.014663219451904297\n",
            "126 0.014673233032226562\n",
            "127 0.018227100372314453\n",
            "Median: 0.011678576469421387\n",
            "PPL: 21103.302734375\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# W4_EIG_CPU\n",
        "\n",
        "# !python opt.py facebook/opt-125m c4 --wbits 4 --nsamples 128 --eigenvalues 1 --save opt-125m-w4-eig-cpu-n128.pt\n",
        "# !python opt.py facebook/opt-125m c4 --load opt-125m-w4-eig-cpu-n128.pt --benchmark 128 --check\n"
      ],
      "metadata": {
        "id": "IaGqzVEaBk_A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "\n",
        "# files.download('/content/gptaq/opt-125m-w4-a4-n128.pt')\n",
        "files.download('/content/gptaq/opt-125m-w4-eig-n128.pt')\n"
      ],
      "metadata": {
        "id": "UpzPntswzXHo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "SO7XMDle9KM1"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "language_info": {
      "name": "python"
    },
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}