molecular-fingerprint-mat.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Molecular Fingerprint MAT .ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1wjJmULckd-Qp2dtW74ctWu7R6JiUUG0Y",
      "authorship_tag": "ABX9TyNdSNyLBV9pJ6ea2yJLJOj8",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/SK124/c01d6699011a27b660675526ca5f1eb5/molecular-fingerprint-mat.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EGzxXVEgmqkW"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8hsvAyk_yku0",
        "outputId": "a26104c8-5b94-4ae6-a0d6-f2ae72584ddf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "\"\"\" Imports neccesary for installing RDkit \"\"\"\n",
        "%cd /content \n",
        "#!wget -c https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh\n",
        "!cp '/content/drive/My Drive/Miniconda3-latest-Linux-x86_64.sh' /content/\n",
        "\n",
        "!chmod +x Miniconda3-latest-Linux-x86_64.sh\n",
        "!time bash ./Miniconda3-latest-Linux-x86_64.sh -b -f -p /usr/local\n",
        "!time conda install -q -y -c conda-forge rdkit\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "import sys\n",
        "import os\n",
        "sys.path.append('/usr/local/lib/python3.7/site-packages/rdkit')\n",
        "%cd /usr/local/lib/python3.7/site-packages/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "PREFIX=/usr/local\n",
            "Unpacking payload ...\n",
            "Collecting package metadata (current_repodata.json): - \b\b\\ \b\bdone\n",
            "Solving environment: / \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\bfailed with initial frozen solve. Retrying with flexible solve.\n",
            "Solving environment: - \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\bfailed with repodata from current_repodata.json, will retry with next repodata source.\n",
            "Collecting package metadata (repodata.json): \\ \b\bdone\n",
            "Solving environment: / \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\bfailed with initial frozen solve. Retrying with flexible solve.\n",
            "Solving environment: | \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \n",
            "Found conflicts! Looking for incompatible packages.\n",
            "This can take several minutes.  Press CTRL-C to abort.\n",
            "\b\bfailed\n",
            "\n",
            "UnsatisfiableError: The following specifications were found to be incompatible with each other:\n",
            "\n",
            "Output in format: Requested package -> Available versions\n",
            "\n",
            "Package xorg-libice conflicts for:\n",
            "xorg-libsm -> xorg-libice=1.0\n",
            "cairo -> xorg-libice\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> xorg-libice\n",
            "cairo -> xorg-libsm -> xorg-libice=1.0\n",
            "xorg-libice\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> xorg-libice\n",
            "\n",
            "Package xz conflicts for:\n",
            "libxml2 -> xz[version='>=5.2.5,<5.3.0a0']\n",
            "pycparser==2.20=py_0 -> python -> xz[version='>=5.2.5,<6.0a0']\n",
            "libtiff -> zstd[version='>=1.4.4,<1.5.0.0a0'] -> xz[version='>=5.2.5,<5.3.0a0']\n",
            "python-dateutil -> python -> xz[version='>=5.2.5,<6.0a0']\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "pandas -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "conda==4.8.3=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "boost-cpp -> xz[version='>=5.2.5,<5.3.0a0']\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "tqdm==4.46.0=py_0 -> python -> xz[version='>=5.2.5,<6.0a0']\n",
            "numpy -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "pytz -> python -> xz[version='>=5.2.5,<6.0a0']\n",
            "pillow -> libtiff[version='>=4.1.0,<5.0a0'] -> xz[version='>=5.2.4,<5.3.0a0|>=5.2.5,<6.0a0']\n",
            "olefile -> python -> xz[version='>=5.2.5,<6.0a0']\n",
            "boost -> boost-cpp=1.72.0 -> xz[version='>=5.2.5,<5.3.0a0|>=5.2.5,<6.0a0']\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "pycairo -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "zstd -> xz[version='>=5.2.5,<5.3.0a0']\n",
            "cffi==1.14.0=py37he30daa8_1 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "urllib3==1.25.8=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "pyopenssl==19.1.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "rdkit -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "fontconfig -> libxml2[version='>=2.9.10,<2.10.0a0'] -> xz[version='>=5.2.5,<5.3.0a0']\n",
            "lcms2 -> libtiff[version='>=4.1.0,<5.0a0'] -> xz[version='>=5.2.4,<5.3.0a0']\n",
            "xz==5.2.5=h7b6447c_0\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "idna==2.9=py_1 -> python -> xz[version='>=5.2.5,<6.0a0']\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "python_abi -> python=3.7 -> xz[version='>=5.2.5,<6.0a0']\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0'] -> xz[version='>=5.2.5,<6.0a0']\n",
            "libtiff -> xz[version='>=5.2.4,<5.3.0a0']\n",
            "python==3.7.7=hcff3b4d_5 -> xz[version='>=5.2.5,<6.0a0']\n",
            "\n",
            "Package ncurses conflicts for:\n",
            "pandas -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "python-dateutil -> python -> ncurses[version='>=6.2,<7.0a0']\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "python_abi -> python=3.7 -> ncurses[version='>=6.2,<7.0a0']\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "python==3.7.7=hcff3b4d_5 -> ncurses[version='>=6.2,<7.0a0']\n",
            "tqdm==4.46.0=py_0 -> python -> ncurses[version='>=6.2,<7.0a0']\n",
            "sqlite==3.31.1=h62c20be_1 -> ncurses[version='>=6.2,<7.0a0']\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "olefile -> python -> ncurses[version='>=6.2,<7.0a0']\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "ncurses==6.2=he6710b0_1\n",
            "sqlite==3.31.1=h62c20be_1 -> libedit[version='>=3.1.20181209,<3.2.0a0'] -> ncurses[version='>=6.1,<7.0a0']\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "boost -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "pytz -> python -> ncurses[version='>=6.2,<7.0a0']\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "urllib3==1.25.8=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "pyopenssl==19.1.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "numpy -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "idna==2.9=py_1 -> python -> ncurses[version='>=6.2,<7.0a0']\n",
            "pillow -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "cffi==1.14.0=py37he30daa8_1 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "pycairo -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "readline==8.0=h7b6447c_0 -> ncurses[version='>=6.1,<7.0a0']\n",
            "conda==4.8.3=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "pycparser==2.20=py_0 -> python -> ncurses[version='>=6.2,<7.0a0']\n",
            "rdkit -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ncurses[version='>=6.2,<7.0a0']\n",
            "libedit==3.1.20181209=hc058e9b_0 -> ncurses[version='>=6.1,<7.0a0']\n",
            "python==3.7.7=hcff3b4d_5 -> readline[version='>=8.0,<9.0a0'] -> ncurses[version='>=6.1,<7.0a0']\n",
            "\n",
            "Package ld_impl_linux-64 conflicts for:\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "idna==2.9=py_1 -> python -> ld_impl_linux-64\n",
            "pandas -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "conda==4.8.3=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "python==3.7.7=hcff3b4d_5 -> ld_impl_linux-64\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "pycairo -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "python_abi -> python=3.7 -> ld_impl_linux-64\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "boost -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "tqdm==4.46.0=py_0 -> python -> ld_impl_linux-64\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "pytz -> python -> ld_impl_linux-64\n",
            "olefile -> python -> ld_impl_linux-64\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "urllib3==1.25.8=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "numpy -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "pyopenssl==19.1.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "cffi==1.14.0=py37he30daa8_1 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "ld_impl_linux-64==2.33.1=h53a641e_7\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "pycparser==2.20=py_0 -> python -> ld_impl_linux-64\n",
            "rdkit -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "pillow -> python[version='>=3.7,<3.8.0a0'] -> ld_impl_linux-64\n",
            "python-dateutil -> python -> ld_impl_linux-64\n",
            "\n",
            "Package zlib conflicts for:\n",
            "pillow -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "conda==4.8.3=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "pandas -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "zstd -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "pycparser==2.20=py_0 -> python -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "cairo -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "python_abi -> python=3.7 -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "lcms2 -> libtiff[version='>=4.1.0,<5.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "python==3.7.7=hcff3b4d_5 -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "sqlite==3.31.1=h62c20be_1 -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "freetype -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "boost-cpp -> zstd[version='>=1.4.4,<1.5.0.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "libtiff -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "olefile -> python -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "tqdm==4.46.0=py_0 -> python -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "pytz -> python -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "urllib3==1.25.8=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "pyopenssl==19.1.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "cffi==1.14.0=py37he30daa8_1 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "glib -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "libpng -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "tk==8.6.8=hbc83047_0 -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "boost -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "numpy -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "libxml2 -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "zlib==1.2.11=h7b6447c_3\n",
            "fontconfig -> freetype[version='>=2.9.1,<3.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "python-dateutil -> python -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "idna==2.9=py_1 -> python -> zlib[version='>=1.2.11,<1.3.0a0']\n",
            "\n",
            "Package _libgcc_mutex conflicts for:\n",
            "boost-cpp -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "libedit==3.1.20181209=hc058e9b_0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "libxml2 -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "fontconfig -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "zstd -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "freetype -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-libxext -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xz==5.2.5=h7b6447c_0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "pycairo -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "cairo -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "libwebp-base -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "libtiff -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "glib -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "python==3.7.7=hcff3b4d_5 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-renderproto -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "openssl==1.1.1g=h7b6447c_0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "icu -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "jpeg -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "cffi==1.14.0=py37he30daa8_1 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "pillow -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "boost -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-kbproto -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "ncurses==6.2=he6710b0_1 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-libsm -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "zlib==1.2.11=h7b6447c_3 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-libxdmcp -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "readline==8.0=h7b6447c_0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "libiconv -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "pthread-stubs -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "libffi==3.3=he6710b0_1 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "libopenblas -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "libpng -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "numpy -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "lz4-c -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "yaml==0.1.7=had09818_2 -> libgcc-ng[version='>=7.2.0'] -> _libgcc_mutex=[build=main]\n",
            "sqlite==3.31.1=h62c20be_1 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-libx11 -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "bzip2 -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-libxau -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "lcms2 -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "tk==8.6.8=hbc83047_0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "_libgcc_mutex==0.1=main\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "rdkit -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "pixman -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "pcre -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "pandas -> libgcc-ng[version='>=7.5.0'] -> _libgcc_mutex=[build=main]\n",
            "libxcb -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-xproto -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-libice -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-libxrender -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "xorg-xextproto -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "libuuid -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]\n",
            "\n",
            "Package openssl conflicts for:\n",
            "pandas -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "pyopenssl==19.1.0=py37_0 -> cryptography[version='>=2.8'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "conda==4.8.3=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "python==3.7.7=hcff3b4d_5 -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "urllib3==1.25.8=py37_0 -> cryptography[version='>=1.3.4'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "pytz -> python -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "python_abi -> python=3.7 -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "openssl==1.1.1g=h7b6447c_0\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "boost -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "tqdm==4.46.0=py_0 -> python -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "rdkit -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "olefile -> python -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "pycairo -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "numpy -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "cffi==1.14.0=py37he30daa8_1 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "pycparser==2.20=py_0 -> python -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "pillow -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "idna==2.9=py_1 -> python -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "python-dateutil -> python -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> openssl[version='>=1.1.1g,<1.1.2a']\n",
            "\n",
            "Package libgcc-ng conflicts for:\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "urllib3==1.25.8=py37_0 -> cryptography[version='>=1.3.4'] -> libgcc-ng[version='>=7.3.0']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> libgcc-ng[version='>=7.3.0']\n",
            "pandas -> libgcc-ng[version='>=7.5.0']\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "openssl==1.1.1g=h7b6447c_0 -> libgcc-ng[version='>=7.3.0']\n",
            "icu -> libgcc-ng[version='>=7.3.0']\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "xorg-libx11 -> libxcb=1 -> libgcc-ng[version='>=7.3.0']\n",
            "idna==2.9=py_1 -> python -> libgcc-ng[version='>=7.3.0']\n",
            "lcms2 -> libtiff[version='>=4.1.0,<5.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "libxcb -> libgcc-ng[version='>=7.3.0']\n",
            "cffi==1.14.0=py37he30daa8_1 -> libgcc-ng[version='>=7.3.0']\n",
            "pyopenssl==19.1.0=py37_0 -> cryptography[version='>=2.8'] -> libgcc-ng[version='>=7.3.0']\n",
            "libblas -> libopenblas[version='>=0.3.10,<0.3.11.0a0'] -> libgcc-ng[version='>=7.5.0']\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "lcms2 -> libgcc-ng[version='>=7.5.0']\n",
            "pycparser==2.20=py_0 -> python -> libgcc-ng[version='>=7.3.0']\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> libgcc-ng[version='>=7.3.0']\n",
            "zlib==1.2.11=h7b6447c_3 -> libgcc-ng[version='>=7.3.0']\n",
            "python==3.7.7=hcff3b4d_5 -> libgcc-ng[version='>=7.3.0']\n",
            "libgcc-ng==9.1.0=hdf63c60_0\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "xorg-libxext -> libgcc-ng[version='>=7.3.0']\n",
            "python_abi -> python=3.7 -> libgcc-ng[version='>=7.3.0']\n",
            "readline==8.0=h7b6447c_0 -> libgcc-ng[version='>=7.3.0']\n",
            "fontconfig -> libpng[version='>=1.6.37,<1.7.0a0'] -> libgcc-ng[version='>=7.5.0']\n",
            "libtiff -> libgcc-ng[version='>=7.3.0']\n",
            "freetype -> libpng[version='>=1.6.37,<1.7.0a0'] -> libgcc-ng[version='>=7.5.0']\n",
            "libuuid -> libgcc-ng[version='>=7.3.0']\n",
            "libwebp-base -> libgcc-ng[version='>=7.3.0']\n",
            "xorg-xextproto -> libgcc-ng[version='>=7.3.0']\n",
            "bzip2 -> libgcc-ng[version='>=7.5.0']\n",
            "numpy -> libgcc-ng[version='>=7.5.0']\n",
            "libxml2 -> libgcc-ng[version='>=7.5.0']\n",
            "rdkit -> boost[version='>=1.72.0,<1.72.1.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "python==3.7.7=hcff3b4d_5 -> openssl[version='>=1.1.1g,<1.1.2a'] -> libgcc-ng[version='>=7.5.0']\n",
            "boost -> boost-cpp=1.72.0 -> libgcc-ng[version='>=7.5.0']\n",
            "xorg-libxrender -> xorg-libx11=1.6 -> libgcc-ng[version='>=7.5.0']\n",
            "pytz -> python -> libgcc-ng[version='>=7.3.0']\n",
            "olefile -> python -> libgcc-ng[version='>=7.3.0']\n",
            "libedit==3.1.20181209=hc058e9b_0 -> libgcc-ng[version='>=7.3.0']\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "ncurses==6.2=he6710b0_1 -> libgcc-ng[version='>=7.3.0']\n",
            "libpng -> libgcc-ng[version='>=7.5.0']\n",
            "xorg-libxdmcp -> libgcc-ng[version='>=7.3.0']\n",
            "xorg-xproto -> libgcc-ng[version='>=7.3.0']\n",
            "boost-cpp -> icu[version='>=67.1,<68.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> libgcc-ng[version='>=7.5.0']\n",
            "yaml==0.1.7=had09818_2 -> libgcc-ng[version='>=7.2.0']\n",
            "libpng -> zlib[version='>=1.2.11,<1.3.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> yaml[version='>=0.1.7,<0.2.0a0'] -> libgcc-ng[version='>=7.2.0']\n",
            "glib -> libgcc-ng[version='>=7.3.0']\n",
            "libffi==3.3=he6710b0_1 -> libgcc-ng[version='>=7.3.0']\n",
            "xorg-renderproto -> libgcc-ng[version='>=7.3.0']\n",
            "freetype -> libgcc-ng[version='>=7.3.0']\n",
            "sqlite==3.31.1=h62c20be_1 -> libgcc-ng[version='>=7.3.0']\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "tqdm==4.46.0=py_0 -> python -> libgcc-ng[version='>=7.3.0']\n",
            "xorg-libsm -> libgcc-ng[version='>=7.3.0']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> openssl[version='>=1.1.1g,<1.1.2a'] -> libgcc-ng[version='>=7.5.0']\n",
            "pcre -> libgcc-ng[version='>=7.3.0']\n",
            "zstd -> libgcc-ng[version='>=7.5.0']\n",
            "pixman -> libgcc-ng[version='>=7.3.0']\n",
            "pthread-stubs -> libgcc-ng[version='>=7.3.0']\n",
            "libtiff -> jpeg[version='>=9c,<10a'] -> libgcc-ng[version='>=7.5.0']\n",
            "xorg-kbproto -> libgcc-ng[version='>=7.3.0']\n",
            "xz==5.2.5=h7b6447c_0 -> libgcc-ng[version='>=7.3.0']\n",
            "boost-cpp -> libgcc-ng[version='>=7.5.0']\n",
            "numpy -> python[version='>=3.7,<3.8.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "pillow -> freetype[version='>=2.9.1,<3.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> libgcc-ng[version='>=7.3.0']\n",
            "lz4-c -> libgcc-ng[version='>=7.5.0']\n",
            "xorg-libx11 -> libgcc-ng[version='>=7.5.0']\n",
            "libiconv -> libgcc-ng[version='>=7.5.0']\n",
            "libxml2 -> icu[version='>=67.1,<68.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "pandas -> python[version='>=3.7,<3.8.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "python-dateutil -> python -> libgcc-ng[version='>=7.3.0']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> libgcc-ng[version='>=7.3.0']\n",
            "cairo -> libgcc-ng[version='>=7.5.0']\n",
            "xorg-libxext -> xorg-libx11=1.6 -> libgcc-ng[version='>=7.5.0']\n",
            "pillow -> libgcc-ng[version='>=7.5.0']\n",
            "boost -> libgcc-ng[version='>=7.3.0']\n",
            "jpeg -> libgcc-ng[version='>=7.5.0']\n",
            "tk==8.6.8=hbc83047_0 -> libgcc-ng[version='>=7.3.0']\n",
            "libopenblas -> libgcc-ng[version='>=7.5.0']\n",
            "conda==4.8.3=py37_0 -> conda-package-handling[version='>=1.3.0'] -> libgcc-ng[version='>=7.3.0']\n",
            "rdkit -> libgcc-ng[version='>=7.5.0']\n",
            "xorg-libice -> libgcc-ng[version='>=7.3.0']\n",
            "xorg-libxrender -> libgcc-ng[version='>=7.3.0']\n",
            "zstd -> xz[version='>=5.2.5,<5.3.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "xorg-libxau -> libgcc-ng[version='>=7.3.0']\n",
            "fontconfig -> libgcc-ng[version='>=7.3.0']\n",
            "pycairo -> libgcc-ng[version='>=7.3.0']\n",
            "cairo -> fontconfig[version='>=2.13.1,<3.0a0'] -> libgcc-ng[version='>=7.3.0']\n",
            "\n",
            "Package xorg-libxdmcp conflicts for:\n",
            "cairo -> libxcb -> xorg-libxdmcp\n",
            "libxcb -> xorg-libxdmcp\n",
            "xorg-libx11 -> libxcb=1 -> xorg-libxdmcp\n",
            "xorg-libxdmcp\n",
            "\n",
            "Package lz4-c conflicts for:\n",
            "libtiff -> zstd[version='>=1.4.4,<1.5.0.0a0'] -> lz4-c[version='>=1.9.2,<1.9.3.0a0']\n",
            "lz4-c\n",
            "zstd -> lz4-c[version='>=1.9.2,<1.9.3.0a0']\n",
            "boost-cpp -> zstd[version='>=1.4.4,<1.5.0.0a0'] -> lz4-c[version='>=1.9.2,<1.9.3.0a0']\n",
            "\n",
            "Package tk conflicts for:\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "boost -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "numpy -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "idna==2.9=py_1 -> python -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "pillow -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "urllib3==1.25.8=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "pycairo -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "pandas -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "rdkit -> pillow -> tk[version='>=8.6.10,<8.7.0a0|>=8.6.8,<8.7.0a0']\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "python-dateutil -> python -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "python_abi -> python=3.7 -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "conda==4.8.3=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "pycparser==2.20=py_0 -> python -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "python==3.7.7=hcff3b4d_5 -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "tqdm==4.46.0=py_0 -> python -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "olefile -> python -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "tk==8.6.8=hbc83047_0\n",
            "pytz -> python -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "pyopenssl==19.1.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "cffi==1.14.0=py37he30daa8_1 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> python[version='>=3.7,<3.8.0a0'] -> tk[version='>=8.6.8,<8.7.0a0']\n",
            "pillow -> tk[version='>=8.6.10,<8.7.0a0']\n",
            "\n",
            "Package libffi conflicts for:\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "pyopenssl==19.1.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "libffi==3.3=he6710b0_1\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "idna==2.9=py_1 -> python -> libffi[version='>=3.3,<3.4.0a0']\n",
            "python_abi -> python=3.7 -> libffi[version='>=3.3,<3.4.0a0']\n",
            "cffi==1.14.0=py37he30daa8_1 -> libffi[version='>=3.3,<3.4.0a0']\n",
            "conda==4.8.3=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "pycparser==2.20=py_0 -> python -> libffi[version='>=3.3,<3.4.0a0']\n",
            "cairo -> glib[version='>=2.58.3,<3.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "python-dateutil -> python -> libffi[version='>=3.3,<3.4.0a0']\n",
            "pandas -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "glib -> libffi[version='>=3.3,<3.4.0a0']\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "python==3.7.7=hcff3b4d_5 -> libffi[version='>=3.3,<3.4.0a0']\n",
            "boost -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "tqdm==4.46.0=py_0 -> python -> libffi[version='>=3.3,<3.4.0a0']\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "olefile -> python -> libffi[version='>=3.3,<3.4.0a0']\n",
            "urllib3==1.25.8=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "pycairo -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "rdkit -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "pillow -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "numpy -> python[version='>=3.7,<3.8.0a0'] -> libffi[version='>=3.3,<3.4.0a0']\n",
            "pytz -> python -> libffi[version='>=3.3,<3.4.0a0']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> cffi -> libffi[version='>=3.3,<3.4.0a0']\n",
            "\n",
            "Package six conflicts for:\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> six\n",
            "rdkit -> six\n",
            "conda==4.8.3=py37_0 -> conda-package-handling[version='>=1.3.0'] -> six[version='>=1.5.2']\n",
            "urllib3==1.25.8=py37_0 -> cryptography[version='>=1.3.4'] -> six[version='>=1.5.2']\n",
            "six==1.14.0=py37_0\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> six\n",
            "pyopenssl==19.1.0=py37_0 -> six[version='>=1.5.2']\n",
            "pyopenssl==19.1.0=py37_0 -> cryptography[version='>=2.8'] -> six\n",
            "pandas -> python-dateutil[version='>=2.7.3'] -> six\n",
            "python-dateutil -> six\n",
            "\n",
            "Package certifi conflicts for:\n",
            "wheel==0.34.2=py37_0 -> setuptools -> certifi[version='>=2016.9.26']\n",
            "conda==4.8.3=py37_0 -> requests[version='>=2.18.4,<3'] -> certifi[version='>=2016.9.26|>=2017.4.17']\n",
            "requests==2.23.0=py37_0 -> urllib3[version='>=1.21.1,<1.26,!=1.25.0,!=1.25.1'] -> certifi\n",
            "pip==20.0.2=py37_3 -> setuptools -> certifi[version='>=2016.9.26']\n",
            "requests==2.23.0=py37_0 -> certifi[version='>=2017.4.17']\n",
            "urllib3==1.25.8=py37_0 -> certifi\n",
            "certifi==2020.4.5.1=py37_0\n",
            "setuptools==46.4.0=py37_0 -> certifi[version='>=2016.9.26']\n",
            "\n",
            "Package python conflicts for:\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "rdkit -> python[version='>=3.7,<3.8.0a0']\n",
            "python==3.7.7=hcff3b4d_5\n",
            "requests==2.23.0=py37_0 -> idna[version='>=2.5,<3'] -> python\n",
            "pyopenssl==19.1.0=py37_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "cffi==1.14.0=py37he30daa8_1 -> pycparser -> python\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0']\n",
            "pycairo -> python_abi=3.7[build=*_cp37m] -> python=3.7\n",
            "pytz -> python\n",
            "urllib3==1.25.8=py37_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "olefile -> python\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> idna -> python\n",
            "conda==4.8.3=py37_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "pycparser==2.20=py_0 -> python\n",
            "tqdm==4.46.0=py_0 -> python\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "numpy -> python[version='>=3.7,<3.8.0a0']\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "pillow -> python[version='>=3.7,<3.8.0a0']\n",
            "python_abi -> python=3.7\n",
            "urllib3==1.25.8=py37_0 -> idna[version='>=2.0.0'] -> python\n",
            "boost -> python[version='>=3.7,<3.8.0a0']\n",
            "rdkit -> python_abi=3.7[build=*_cp37m] -> python=3.7\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> tqdm -> python\n",
            "pandas -> python[version='>=3.7,<3.8.0a0']\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "pillow -> olefile -> python=3.7\n",
            "idna==2.9=py_1 -> python\n",
            "pycairo -> python[version='>=3.7,<3.8.0a0']\n",
            "numpy -> python_abi=3.7[build=*_cp37m] -> python=3.7\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "cffi==1.14.0=py37he30daa8_1 -> python[version='>=3.7,<3.8.0a0']\n",
            "pandas -> python-dateutil[version='>=2.7.3'] -> python=3.7\n",
            "python-dateutil -> six -> python[version='>=3.7,<3.8.0a0']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0']\n",
            "python-dateutil -> python\n",
            "\n",
            "Package libstdcxx-ng conflicts for:\n",
            "boost-cpp -> icu[version='>=67.1,<68.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "readline==8.0=h7b6447c_0 -> ncurses[version='>=6.1,<7.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "libxml2 -> icu[version='>=67.1,<68.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "libffi==3.3=he6710b0_1 -> libstdcxx-ng[version='>=7.3.0']\n",
            "cffi==1.14.0=py37he30daa8_1 -> libffi[version='>=3.3,<3.4.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "python==3.7.7=hcff3b4d_5 -> libffi[version='>=3.3,<3.4.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "icu -> libstdcxx-ng[version='>=7.3.0']\n",
            "rdkit -> libstdcxx-ng[version='>=7.5.0']\n",
            "fontconfig -> icu[version='>=67.1,<68.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "boost -> libstdcxx-ng[version='>=7.3.0']\n",
            "boost -> boost-cpp=1.72.0 -> libstdcxx-ng[version='>=7.5.0']\n",
            "lcms2 -> libtiff[version='>=4.1.0,<5.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "cairo -> glib[version='>=2.58.3,<3.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "rdkit -> boost[version='>=1.72.0,<1.72.1.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "sqlite==3.31.1=h62c20be_1 -> ncurses[version='>=6.2,<7.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "libtiff -> libstdcxx-ng[version='>=7.3.0']\n",
            "zstd -> libstdcxx-ng[version='>=7.5.0']\n",
            "pandas -> libstdcxx-ng[version='>=7.5.0']\n",
            "lz4-c -> libstdcxx-ng[version='>=7.5.0']\n",
            "boost-cpp -> libstdcxx-ng[version='>=7.5.0']\n",
            "glib -> libstdcxx-ng[version='>=7.3.0']\n",
            "libtiff -> zstd[version='>=1.4.4,<1.5.0.0a0'] -> libstdcxx-ng[version='>=7.5.0']\n",
            "pillow -> libtiff[version='>=4.1.0,<5.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "libstdcxx-ng==9.1.0=hdf63c60_0\n",
            "libedit==3.1.20181209=hc058e9b_0 -> ncurses[version='>=6.1,<7.0a0'] -> libstdcxx-ng[version='>=7.3.0']\n",
            "pcre -> libstdcxx-ng[version='>=7.3.0']\n",
            "ncurses==6.2=he6710b0_1 -> libstdcxx-ng[version='>=7.3.0']\n",
            "\n",
            "Package libgfortran-ng conflicts for:\n",
            "libgfortran-ng\n",
            "libopenblas -> libgfortran-ng[version='>=7,<8.0a0']\n",
            "libblas -> libopenblas[version='>=0.3.10,<0.3.11.0a0'] -> libgfortran-ng[version='>=7,<8.0a0']\n",
            "\n",
            "Package glib conflicts for:\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> glib[version='>=2.58.3,<3.0a0']\n",
            "glib\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> glib[version='>=2.58.3,<3.0a0']\n",
            "cairo -> glib[version='>=2.58.3,<3.0a0']\n",
            "\n",
            "Package xorg-renderproto conflicts for:\n",
            "xorg-renderproto\n",
            "cairo -> xorg-libxrender -> xorg-renderproto\n",
            "xorg-libxrender -> xorg-renderproto\n",
            "\n",
            "Package libedit conflicts for:\n",
            "sqlite==3.31.1=h62c20be_1 -> libedit[version='>=3.1.20181209,<3.2.0a0']\n",
            "python==3.7.7=hcff3b4d_5 -> sqlite[version='>=3.31.1,<4.0a0'] -> libedit[version='>=3.1.20181209,<3.2.0a0']\n",
            "libedit==3.1.20181209=hc058e9b_0\n",
            "\n",
            "Package libblas conflicts for:\n",
            "rdkit -> numpy[version='>=1.16.5,<2.0a0'] -> libblas[version='>=3.8.0,<4.0a0']\n",
            "numpy -> libblas[version='>=3.8.0,<4.0a0']\n",
            "boost -> numpy[version='>=1.14.6,<2.0a0'] -> libblas[version='>=3.8.0,<4.0a0']\n",
            "libcblas -> libblas==3.8.0=17_openblas\n",
            "libblas\n",
            "liblapack -> libblas==3.8.0=17_openblas\n",
            "numpy -> libcblas[version='>=3.8.0,<4.0a0'] -> libblas==3.8.0=17_openblas\n",
            "pandas -> numpy[version='>=1.15.4,<2.0a0'] -> libblas[version='>=3.8.0,<4.0a0']\n",
            "\n",
            "Package numpy conflicts for:\n",
            "rdkit -> numpy[version='>=1.16.5,<2.0a0']\n",
            "numpy\n",
            "rdkit -> boost[version='>=1.72.0,<1.72.1.0a0'] -> numpy[version='>=1.14.6,<2.0a0|>=1.15.4,<2.0a0']\n",
            "boost -> numpy[version='>=1.14.6,<2.0a0']\n",
            "pandas -> numpy[version='>=1.15.4,<2.0a0']\n",
            "\n",
            "Package libpng conflicts for:\n",
            "freetype -> libpng[version='>=1.6.37,<1.7.0a0']\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> libpng[version='>=1.6.37,<1.7.0a0']\n",
            "pillow -> freetype[version='>=2.9.1,<3.0a0'] -> libpng[version='>=1.6.37,<1.7.0a0']\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> libpng[version='>=1.6.37,<1.7.0a0']\n",
            "fontconfig -> libpng[version='>=1.6.37,<1.7.0a0']\n",
            "cairo -> libpng[version='>=1.6.37,<1.7.0a0']\n",
            "libpng\n",
            "\n",
            "Package setuptools conflicts for:\n",
            "pip==20.0.2=py37_3 -> setuptools\n",
            "setuptools==46.4.0=py37_0\n",
            "wheel==0.34.2=py37_0 -> setuptools\n",
            "conda==4.8.3=py37_0 -> setuptools[version='>=31.0.1']\n",
            "\n",
            "Package readline conflicts for:\n",
            "cffi==1.14.0=py37he30daa8_1 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "python==3.7.7=hcff3b4d_5 -> readline[version='>=8.0,<9.0a0']\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "boost -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "tqdm==4.46.0=py_0 -> python -> readline[version='>=8.0,<9.0a0']\n",
            "rdkit -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "pycairo -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "olefile -> python -> readline[version='>=8.0,<9.0a0']\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "urllib3==1.25.8=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "pillow -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "numpy -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "pyopenssl==19.1.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "python-dateutil -> python -> readline[version='>=8.0,<9.0a0']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "idna==2.9=py_1 -> python -> readline[version='>=8.0,<9.0a0']\n",
            "conda==4.8.3=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "pycparser==2.20=py_0 -> python -> readline[version='>=8.0,<9.0a0']\n",
            "readline==8.0=h7b6447c_0\n",
            "pandas -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "python_abi -> python=3.7 -> readline[version='>=8.0,<9.0a0']\n",
            "pytz -> python -> readline[version='>=8.0,<9.0a0']\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> readline[version='>=8.0,<9.0a0']\n",
            "\n",
            "Package pysocks conflicts for:\n",
            "pysocks==1.7.1=py37_0\n",
            "urllib3==1.25.8=py37_0 -> pysocks[version='>=1.5.6,<2.0,!=1.5.7']\n",
            "requests==2.23.0=py37_0 -> urllib3[version='>=1.21.1,<1.26,!=1.25.0,!=1.25.1'] -> pysocks[version='>=1.5.6,<2.0,!=1.5.7']\n",
            "\n",
            "Package xorg-kbproto conflicts for:\n",
            "xorg-libxrender -> xorg-libx11=1.6 -> xorg-kbproto\n",
            "xorg-kbproto\n",
            "cairo -> xorg-libx11 -> xorg-kbproto\n",
            "xorg-libx11 -> xorg-kbproto\n",
            "xorg-libxext -> xorg-libx11=1.6 -> xorg-kbproto\n",
            "\n",
            "Package libiconv conflicts for:\n",
            "libiconv\n",
            "fontconfig -> libxml2[version='>=2.9.10,<2.10.0a0'] -> libiconv[version='>=1.16,<1.17.0a0']\n",
            "libxml2 -> libiconv[version='>=1.16,<1.17.0a0']\n",
            "\n",
            "Package zstd conflicts for:\n",
            "libtiff -> zstd[version='>=1.4.4,<1.5.0.0a0']\n",
            "pillow -> libtiff[version='>=4.1.0,<5.0a0'] -> zstd[version='>=1.4.4,<1.5.0.0a0']\n",
            "boost -> boost-cpp=1.72.0 -> zstd[version='>=1.4.4,<1.5.0.0a0']\n",
            "zstd\n",
            "lcms2 -> libtiff[version='>=4.1.0,<5.0a0'] -> zstd[version='>=1.4.4,<1.5.0.0a0']\n",
            "boost-cpp -> zstd[version='>=1.4.4,<1.5.0.0a0']\n",
            "\n",
            "Package libxml2 conflicts for:\n",
            "libxml2\n",
            "cairo -> fontconfig[version='>=2.13.1,<3.0a0'] -> libxml2[version='>=2.9.10,<2.10.0a0']\n",
            "fontconfig -> libxml2[version='>=2.9.10,<2.10.0a0']\n",
            "\n",
            "Package sqlite conflicts for:\n",
            "tqdm==4.46.0=py_0 -> python -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "pycosat==0.6.3=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "numpy -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "pytz -> python -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "pycairo -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "six==1.14.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "urllib3==1.25.8=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "pyopenssl==19.1.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "sqlite==3.31.1=h62c20be_1\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "cffi==1.14.0=py37he30daa8_1 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "boost -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "pycparser==2.20=py_0 -> python -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "chardet==3.0.4=py37_1003 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "rdkit -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "idna==2.9=py_1 -> python -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "pillow -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "pandas -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "python_abi -> python=3.7 -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "conda==4.8.3=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "setuptools==46.4.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "pip==20.0.2=py37_3 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "python==3.7.7=hcff3b4d_5 -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "pysocks==1.7.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "requests==2.23.0=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "certifi==2020.4.5.1=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "python-dateutil -> python -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "wheel==0.34.2=py37_0 -> python[version='>=3.7,<3.8.0a0'] -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "olefile -> python -> sqlite[version='>=3.31.1,<4.0a0']\n",
            "\n",
            "Package bzip2 conflicts for:\n",
            "bzip2\n",
            "boost -> boost-cpp=1.72.0 -> bzip2[version='>=1.0.8,<2.0a0']\n",
            "boost-cpp -> bzip2[version='>=1.0.8,<2.0a0']\n",
            "\n",
            "Package python_abi conflicts for:\n",
            "pandas -> python_abi=3.7[build=*_cp37m]\n",
            "requests==2.23.0=py37_0 -> certifi[version='>=2017.4.17'] -> python_abi=3.7[build=*_cp37m]\n",
            "pycairo -> python_abi=3.7[build=*_cp37m]\n",
            "pillow -> python_abi=3.7[build=*_cp37m]\n",
            "boost -> numpy[version='>=1.14.6,<2.0a0'] -> python_abi=3.7[build=*_cp37m]\n",
            "rdkit -> python_abi=3.7[build=*_cp37m]\n",
            "python_abi\n",
            "urllib3==1.25.8=py37_0 -> certifi -> python_abi=3.7[build=*_cp37m]\n",
            "numpy -> python_abi=3.7[build=*_cp37m]\n",
            "setuptools==46.4.0=py37_0 -> certifi[version='>=2016.9.26'] -> python_abi=3.7[build=*_cp37m]\n",
            "\n",
            "Package xorg-xextproto conflicts for:\n",
            "xorg-xextproto\n",
            "cairo -> xorg-libxext -> xorg-xextproto\n",
            "xorg-libxext -> xorg-xextproto\n",
            "\n",
            "Package libxcb conflicts for:\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> libxcb\n",
            "xorg-libxrender -> xorg-libx11=1.6 -> libxcb=1\n",
            "libxcb\n",
            "xorg-libx11 -> libxcb=1\n",
            "cairo -> xorg-libx11 -> libxcb=1\n",
            "xorg-libxext -> xorg-libx11=1.6 -> libxcb=1\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> libxcb\n",
            "cairo -> libxcb\n",
            "\n",
            "Package chardet conflicts for:\n",
            "chardet==3.0.4=py37_1003\n",
            "conda==4.8.3=py37_0 -> requests[version='>=2.18.4,<3'] -> chardet[version='>=3.0.2,<4']\n",
            "requests==2.23.0=py37_0 -> chardet[version='>=3.0.2,<4']\n",
            "\n",
            "Package freetype conflicts for:\n",
            "pillow -> freetype[version='>=2.9.1,<3.0a0']\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> freetype[version='>=2.9.1,<3.0a0']\n",
            "freetype\n",
            "cairo -> freetype[version='>=2.9.1,<3.0a0']\n",
            "fontconfig -> freetype[version='>=2.9.1,<3.0a0']\n",
            "rdkit -> freetype[version='>=2.9.1,<3.0a0']\n",
            "\n",
            "Package pytz conflicts for:\n",
            "rdkit -> pandas -> pytz[version='>=2017.2']\n",
            "pandas -> pytz[version='>=2017.2']\n",
            "pytz\n",
            "\n",
            "Package icu conflicts for:\n",
            "fontconfig -> icu[version='>=67.1,<68.0a0']\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> icu[version='>=67.1,<68.0a0']\n",
            "libxml2 -> icu[version='>=67.1,<68.0a0']\n",
            "cairo -> icu[version='>=67.1,<68.0a0']\n",
            "boost -> boost-cpp=1.72.0 -> icu[version='>=67.1,<68.0a0']\n",
            "icu\n",
            "boost-cpp -> icu[version='>=67.1,<68.0a0']\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> icu[version='>=67.1,<68.0a0']\n",
            "\n",
            "Package libtiff conflicts for:\n",
            "rdkit -> pillow -> libtiff[version='>=4.1.0,<5.0a0']\n",
            "lcms2 -> libtiff[version='>=4.1.0,<5.0a0']\n",
            "pillow -> libtiff[version='>=4.1.0,<5.0a0']\n",
            "libtiff\n",
            "\n",
            "Package python-dateutil conflicts for:\n",
            "pandas -> python-dateutil[version='>=2.7.3']\n",
            "python-dateutil\n",
            "rdkit -> pandas -> python-dateutil[version='>=2.7.3']\n",
            "\n",
            "Package liblapack conflicts for:\n",
            "boost -> numpy[version='>=1.14.6,<2.0a0'] -> liblapack[version='>=3.8.0,<3.9.0a0']\n",
            "numpy -> liblapack[version='>=3.8.0,<3.9.0a0']\n",
            "liblapack\n",
            "pandas -> numpy[version='>=1.15.4,<2.0a0'] -> liblapack[version='>=3.8.0,<3.9.0a0']\n",
            "rdkit -> numpy[version='>=1.16.5,<2.0a0'] -> liblapack[version='>=3.8.0,<3.9.0a0']\n",
            "\n",
            "Package fontconfig conflicts for:\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> fontconfig[version='>=2.13.1,<3.0a0']\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> fontconfig[version='>=2.13.1,<3.0a0']\n",
            "fontconfig\n",
            "cairo -> fontconfig[version='>=2.13.1,<3.0a0']\n",
            "\n",
            "Package libwebp-base conflicts for:\n",
            "libwebp-base\n",
            "lcms2 -> libtiff[version='>=4.1.0,<5.0a0'] -> libwebp-base[version='>=1.1.0,<1.2.0a0']\n",
            "pillow -> libtiff[version='>=4.1.0,<5.0a0'] -> libwebp-base[version='>=1.1.0,<1.2.0a0']\n",
            "libtiff -> libwebp-base[version='>=1.1.0,<1.2.0a0']\n",
            "\n",
            "Package pyopenssl conflicts for:\n",
            "conda==4.8.3=py37_0 -> pyopenssl[version='>=16.2.0']\n",
            "pyopenssl==19.1.0=py37_0\n",
            "requests==2.23.0=py37_0 -> urllib3[version='>=1.21.1,<1.26,!=1.25.0,!=1.25.1'] -> pyopenssl[version='>=0.14']\n",
            "urllib3==1.25.8=py37_0 -> pyopenssl[version='>=0.14']\n",
            "\n",
            "Package xorg-xproto conflicts for:\n",
            "xorg-xproto\n",
            "xorg-libxrender -> xorg-libx11=1.6 -> xorg-xproto\n",
            "xorg-libx11 -> xorg-xproto\n",
            "xorg-libxext -> xorg-libx11=1.6 -> xorg-xproto\n",
            "cairo -> xorg-libx11 -> xorg-xproto\n",
            "\n",
            "Package ca-certificates conflicts for:\n",
            "python==3.7.7=hcff3b4d_5 -> openssl[version='>=1.1.1g,<1.1.2a'] -> ca-certificates\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> openssl[version='>=1.1.1g,<1.1.2a'] -> ca-certificates\n",
            "openssl==1.1.1g=h7b6447c_0 -> ca-certificates\n",
            "ca-certificates==2020.1.1=0\n",
            "\n",
            "Package xorg-libxrender conflicts for:\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> xorg-libxrender\n",
            "cairo -> xorg-libxrender\n",
            "xorg-libxrender\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> xorg-libxrender\n",
            "\n",
            "Package pycparser conflicts for:\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> cffi -> pycparser\n",
            "cffi==1.14.0=py37he30daa8_1 -> pycparser\n",
            "pycparser==2.20=py_0\n",
            "\n",
            "Package idna conflicts for:\n",
            "requests==2.23.0=py37_0 -> urllib3[version='>=1.21.1,<1.26,!=1.25.0,!=1.25.1'] -> idna[version='>=2.0.0']\n",
            "urllib3==1.25.8=py37_0 -> idna[version='>=2.0.0']\n",
            "idna==2.9=py_1\n",
            "urllib3==1.25.8=py37_0 -> cryptography[version='>=1.3.4'] -> idna\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> idna\n",
            "pyopenssl==19.1.0=py37_0 -> cryptography[version='>=2.8'] -> idna\n",
            "requests==2.23.0=py37_0 -> idna[version='>=2.5,<3']\n",
            "conda==4.8.3=py37_0 -> requests[version='>=2.18.4,<3'] -> idna[version='>=2.5,<3']\n",
            "\n",
            "Package xorg-libsm conflicts for:\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> xorg-libsm\n",
            "cairo -> xorg-libsm\n",
            "xorg-libsm\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> xorg-libsm\n",
            "\n",
            "Package pixman conflicts for:\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> pixman[version='>=0.38.0,<0.39.0a0']\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> pixman[version='>=0.38.0,<0.39.0a0']\n",
            "pixman\n",
            "cairo -> pixman[version='>=0.38.0,<0.39.0a0']\n",
            "\n",
            "Package cryptography conflicts for:\n",
            "conda==4.8.3=py37_0 -> pyopenssl[version='>=16.2.0'] -> cryptography[version='>=2.8']\n",
            "cryptography==2.9.2=py37h1ba5d50_0\n",
            "urllib3==1.25.8=py37_0 -> cryptography[version='>=1.3.4']\n",
            "urllib3==1.25.8=py37_0 -> pyopenssl[version='>=0.14'] -> cryptography[version='>=2.8']\n",
            "requests==2.23.0=py37_0 -> urllib3[version='>=1.21.1,<1.26,!=1.25.0,!=1.25.1'] -> cryptography[version='>=1.3.4']\n",
            "pyopenssl==19.1.0=py37_0 -> cryptography[version='>=2.8']\n",
            "\n",
            "Package ruamel_yaml conflicts for:\n",
            "conda==4.8.3=py37_0 -> ruamel_yaml[version='>=0.11.14,<0.16']\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0\n",
            "\n",
            "Package pillow conflicts for:\n",
            "pillow\n",
            "rdkit -> pillow\n",
            "\n",
            "Package boost conflicts for:\n",
            "boost\n",
            "rdkit -> boost[version='>=1.72.0,<1.72.1.0a0']\n",
            "\n",
            "Package pandas conflicts for:\n",
            "pandas\n",
            "rdkit -> pandas\n",
            "\n",
            "Package pthread-stubs conflicts for:\n",
            "pthread-stubs\n",
            "cairo -> libxcb -> pthread-stubs\n",
            "libxcb -> pthread-stubs\n",
            "xorg-libx11 -> libxcb=1 -> pthread-stubs\n",
            "\n",
            "Package cairo conflicts for:\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0']\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0']\n",
            "cairo\n",
            "\n",
            "Package xorg-libxext conflicts for:\n",
            "cairo -> xorg-libxext\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> xorg-libxext\n",
            "xorg-libxext\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> xorg-libxext\n",
            "\n",
            "Package cffi conflicts for:\n",
            "cffi==1.14.0=py37he30daa8_1\n",
            "pyopenssl==19.1.0=py37_0 -> cryptography[version='>=2.8'] -> cffi\n",
            "urllib3==1.25.8=py37_0 -> cryptography[version='>=1.3.4'] -> cffi\n",
            "cryptography==2.9.2=py37h1ba5d50_0 -> cffi\n",
            "\n",
            "Package libuuid conflicts for:\n",
            "libuuid\n",
            "xorg-libsm -> libuuid[version='>=2.32.1,<3.0a0']\n",
            "fontconfig -> libuuid[version='>=2.32.1,<3.0a0']\n",
            "cairo -> fontconfig[version='>=2.13.1,<3.0a0'] -> libuuid[version='>=2.32.1,<3.0a0']\n",
            "\n",
            "Package jpeg conflicts for:\n",
            "lcms2 -> libtiff[version='>=4.1.0,<5.0a0'] -> jpeg[version='>=9c,<10a']\n",
            "rdkit -> pillow -> jpeg[version='>=9d,<10a']\n",
            "libtiff -> jpeg[version='>=9c,<10a']\n",
            "jpeg\n",
            "pillow -> libtiff[version='>=4.1.0,<5.0a0'] -> jpeg[version='>=9c,<10a']\n",
            "pillow -> jpeg[version='>=9d,<10a']\n",
            "lcms2 -> jpeg[version='>=9d,<10a']\n",
            "\n",
            "Package yaml conflicts for:\n",
            "conda==4.8.3=py37_0 -> ruamel_yaml[version='>=0.11.14,<0.16'] -> yaml[version='>=0.1.7,<0.2.0a0']\n",
            "yaml==0.1.7=had09818_2\n",
            "ruamel_yaml==0.15.87=py37h7b6447c_0 -> yaml[version='>=0.1.7,<0.2.0a0']\n",
            "\n",
            "Package conda-package-handling conflicts for:\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0\n",
            "conda==4.8.3=py37_0 -> conda-package-handling[version='>=1.3.0']\n",
            "\n",
            "Package xorg-libx11 conflicts for:\n",
            "cairo -> xorg-libx11\n",
            "xorg-libx11\n",
            "cairo -> xorg-libxext -> xorg-libx11=1.6\n",
            "rdkit -> cairo[version='>=1.16.0,<1.17.0a0'] -> xorg-libx11\n",
            "xorg-libxrender -> xorg-libx11=1.6\n",
            "xorg-libxext -> xorg-libx11=1.6\n",
            "pycairo -> cairo[version='>=1.16.0,<1.17.0a0'] -> xorg-libx11\n",
            "\n",
            "Package lcms2 conflicts for:\n",
            "pillow -> lcms2[version='>=2.11,<3.0a0']\n",
            "rdkit -> pillow -> lcms2[version='>=2.11,<3.0a0']\n",
            "lcms2\n",
            "\n",
            "Package tqdm conflicts for:\n",
            "conda-package-handling==1.6.1=py37h7b6447c_0 -> tqdm\n",
            "tqdm==4.46.0=py_0\n",
            "conda==4.8.3=py37_0 -> conda-package-handling[version='>=1.3.0'] -> tqdm\n",
            "\n",
            "Package pcre conflicts for:\n",
            "cairo -> glib[version='>=2.58.3,<3.0a0'] -> pcre[version='>=8.44,<9.0a0']\n",
            "pcre\n",
            "glib -> pcre[version='>=8.44,<9.0a0']\n",
            "\n",
            "Package xorg-libxau conflicts for:\n",
            "cairo -> libxcb -> xorg-libxau\n",
            "xorg-libx11 -> libxcb=1 -> xorg-libxau\n",
            "libxcb -> xorg-libxau\n",
            "xorg-libxau\n",
            "\n",
            "Package libcblas conflicts for:\n",
            "rdkit -> numpy[version='>=1.16.5,<2.0a0'] -> libcblas[version='>=3.8.0,<4.0a0']\n",
            "numpy -> libcblas[version='>=3.8.0,<4.0a0']\n",
            "boost -> numpy[version='>=1.14.6,<2.0a0'] -> libcblas[version='>=3.8.0,<4.0a0']\n",
            "libcblas\n",
            "pandas -> numpy[version='>=1.15.4,<2.0a0'] -> libcblas[version='>=3.8.0,<4.0a0']\n",
            "\n",
            "Package boost-cpp conflicts for:\n",
            "boost -> boost-cpp=1.72.0\n",
            "boost-cpp\n",
            "rdkit -> boost[version='>=1.72.0,<1.72.1.0a0'] -> boost-cpp=1.72.0\n",
            "\n",
            "Package wheel conflicts for:\n",
            "pip==20.0.2=py37_3 -> wheel\n",
            "wheel==0.34.2=py37_0\n",
            "\n",
            "Package libopenblas conflicts for:\n",
            "libblas -> libopenblas[version='>=0.3.10,<0.3.11.0a0|>=0.3.10,<1.0a0']\n",
            "liblapack -> libblas==3.8.0=17_openblas -> libopenblas[version='>=0.3.10,<0.3.11.0a0|>=0.3.10,<1.0a0']\n",
            "libcblas -> libblas==3.8.0=17_openblas -> libopenblas[version='>=0.3.10,<0.3.11.0a0|>=0.3.10,<1.0a0']\n",
            "numpy -> libblas[version='>=3.8.0,<4.0a0'] -> libopenblas[version='>=0.3.10,<0.3.11.0a0|>=0.3.10,<1.0a0']\n",
            "libopenblas\n",
            "\n",
            "Package urllib3 conflicts for:\n",
            "conda==4.8.3=py37_0 -> requests[version='>=2.18.4,<3'] -> urllib3[version='>=1.21.1,<1.26,!=1.25.0,!=1.25.1']\n",
            "requests==2.23.0=py37_0 -> urllib3[version='>=1.21.1,<1.26,!=1.25.0,!=1.25.1']\n",
            "urllib3==1.25.8=py37_0\n",
            "\n",
            "Package pycairo conflicts for:\n",
            "pycairo\n",
            "rdkit -> pycairo\n",
            "\n",
            "Package requests conflicts for:\n",
            "conda==4.8.3=py37_0 -> requests[version='>=2.18.4,<3']\n",
            "requests==2.23.0=py37_0\n",
            "\n",
            "Package olefile conflicts for:\n",
            "pillow -> olefile\n",
            "olefile\n",
            "rdkit -> pillow -> olefile\n",
            "\n",
            "Package pycosat conflicts for:\n",
            "conda==4.8.3=py37_0 -> pycosat[version='>=0.6.3']\n",
            "pycosat==0.6.3=py37h7b6447c_0\n",
            "\n",
            "\n",
            "real\t0m41.393s\n",
            "user\t0m51.555s\n",
            "sys\t0m6.550s\n",
            "Collecting package metadata (current_repodata.json): ...working... done\n",
            "Solving environment: ...working... done\n",
            "\n",
            "# All requested packages already installed.\n",
            "\n",
            "\n",
            "real\t0m9.325s\n",
            "user\t0m8.461s\n",
            "sys\t0m0.939s\n",
            "/usr/local/lib/python3.7/site-packages\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NH_TzY5Nf4hx",
        "outputId": "7d241c4f-8122-4f20-e1b7-e6c4752f4050",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%cd /usr/local/lib/python3.7/site-packages/\n",
        "from rdkit import Chem\n",
        "from rdkit.Chem import AllChem\n",
        "from rdkit.Chem import MolFromSmiles"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/site-packages\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fBFcXrmfQSXa",
        "outputId": "c000b466-40a0-4356-cef0-6c4013978793",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "%cd /content\n",
        "!git clone https://github.com/ardigen/MAT.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "Cloning into 'MAT'...\n",
            "remote: Enumerating objects: 101, done.\u001b[K\n",
            "remote: Total 101 (delta 0), reused 0 (delta 0), pack-reused 101\u001b[K\n",
            "Receiving objects: 100% (101/101), 414.45 KiB | 5.53 MiB/s, done.\n",
            "Resolving deltas: 100% (23/23), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RFEnHjn8gAWq"
      },
      "source": [
        "\n",
        "import os\n",
        "import pandas as pd\n",
        "import torch\n",
        "from rdkit import Chem\n",
        "from rdkit.Chem import AllChem\n",
        "from rdkit.Chem import MolFromSmiles\n",
        "#os.chdir('src')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZO-NhZh_Qy20",
        "outputId": "b7d2cbb8-ff32-4458-d25a-11a3287d2f75",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%cd /content/MAT\n",
        "os.chdir('src')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/MAT\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xm8lEFq5yXeC"
      },
      "source": [
        "from featurization.data_utils import load_data_from_df, construct_loader"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y0Xgg1wI3u-X"
      },
      "source": [
        "batch_size = 64\n",
        "\n",
        "# Formal charges are one-hot encoded to keep compatibility with the pre-trained weights.\n",
        "# If you do not plan to use the pre-trained weights, we recommend to set one_hot_formal_charge to False.\n",
        "X, y = load_data_from_df('../data/freesolv/freesolv.csv', one_hot_formal_charge=True)\n",
        "data_loader = construct_loader(X, y, batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FBEa-UHv4fri",
        "outputId": "4bc5abf4-8d0d-406f-9f90-f97122c5e236",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        }
      },
      "source": [
        "pd.read_csv('../data/freesolv/freesolv.csv').head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>smiles</th>\n",
              "      <th>y</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CN(C)C(=O)c1ccc(cc1)OC</td>\n",
              "      <td>-1.874467</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CS(=O)(=O)Cl</td>\n",
              "      <td>-0.277514</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CC(C)C=C</td>\n",
              "      <td>1.465089</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CCc1cnccn1</td>\n",
              "      <td>-0.428367</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CCCCCCCO</td>\n",
              "      <td>-0.105855</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   smiles         y\n",
              "0  CN(C)C(=O)c1ccc(cc1)OC -1.874467\n",
              "1            CS(=O)(=O)Cl -0.277514\n",
              "2                CC(C)C=C  1.465089\n",
              "3              CCc1cnccn1 -0.428367\n",
              "4                CCCCCCCO -0.105855"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nu8Cqjuh7Kad"
      },
      "source": [
        "df=pd.read_csv('../data/bbbp/bbbp.csv').head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FZEWkNmr7Sy_"
      },
      "source": [
        "df=df[:128]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uwjixHuI7XNt"
      },
      "source": [
        "batch_size = 32\n",
        "\n",
        "# Formal charges are one-hot encoded to keep compatibility with the pre-trained weights.\n",
        "# If you do not plan to use the pre-trained weights, we recommend to set one_hot_formal_charge to False.\n",
        "X, y = load_data_from_df('../data/freesolv/freesolv.csv', one_hot_formal_charge=True)\n",
        "data_loader = construct_loader(X, y, batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jkNk1ouv7rcE"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "do3OkOdr7bnj"
      },
      "source": [
        "df.to_csv(r'/content/MAT/data/bbbp/bbbp128.csv', index = False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aqqv3PVV4jr4"
      },
      "source": [
        "from transformer import make_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wNS8LXxD6U_A",
        "outputId": "033e9531-5b64-4ae8-c8d8-18d96d440fbc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "X[4][0].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(9, 28)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qcfjjv4m6QAx",
        "outputId": "5f8cd726-0454-419e-e356-02bba5f2d280",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "d_atom = X[0][0].shape[1]\n",
        "d_atom"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "28"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uOmXvC5Ucbpo",
        "outputId": "a8aaaea4-a932-4f79-b18b-fbe8a91df426",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "(X[641][0]).shape[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "28"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ix7mVh2Mt_sb"
      },
      "source": [
        "en\n",
        "import math, copy\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "\n",
        "from utils import xavier_normal_small_init_, xavier_uniform_small_init_\n",
        "\n",
        "\n",
        "### Model definition\n",
        "\n",
        "def make_models(d_atom, N=2, d_model=128, h=8, dropout=0.1, \n",
        "               lambda_attention=0.3, lambda_distance=0.3, trainable_lambda=False,\n",
        "               N_dense=2, leaky_relu_slope=0.0, aggregation_type='mean', \n",
        "               dense_output_nonlinearity='relu', distance_matrix_kernel='softmax',\n",
        "               use_edge_features=False, n_output=1,\n",
        "               control_edges=False, integrated_distances=False, \n",
        "               scale_norm=False, init_type='uniform', use_adapter=False, n_generator_layers=1):\n",
        "    \"Helper: Construct a model from hyperparameters.\"\n",
        "    c = copy.deepcopy\n",
        "    attn = MultiHeadedAttention(h, d_model, dropout, lambda_attention, lambda_distance, trainable_lambda, distance_matrix_kernel, use_edge_features, control_edges, integrated_distances)\n",
        "    ff = PositionwiseFeedForward(d_model, N_dense, dropout, leaky_relu_slope, dense_output_nonlinearity)\n",
        "    model = GraphTransformer(\n",
        "        Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout, scale_norm, use_adapter), N, scale_norm),\n",
        "        Embeddings(d_model, d_atom, dropout))\n",
        "       # Generator(d_model, aggregation_type, n_output, n_generator_layers, leaky_relu_slope, dropout, scale_norm))\n",
        "    \n",
        "    # This was important from their code. \n",
        "    # Initialize parameters with Glorot / fan_avg.\n",
        "    for p in model.parameters():\n",
        "        if p.dim() > 1:\n",
        "            if init_type == 'uniform':\n",
        "                nn.init.xavier_uniform_(p)\n",
        "            elif init_type == 'normal':\n",
        "                nn.init.xavier_normal_(p)\n",
        "            elif init_type == 'small_normal_init':\n",
        "                xavier_normal_small_init_(p)\n",
        "            elif init_type == 'small_uniform_init':\n",
        "                xavier_uniform_small_init_(p)\n",
        "    return model\n",
        "\n",
        "\n",
        "class GraphTransformer(nn.Module):\n",
        "    def __init__(self, encoder, src_embed):\n",
        "        super(GraphTransformer, self).__init__()\n",
        "        self.encoder = encoder\n",
        "        self.src_embed = src_embed\n",
        "        \n",
        "        \n",
        "    def forward(self, src, src_mask, adj_matrix, distances_matrix, edges_att):\n",
        "        \"Take in and process masked src and target sequences.\"\n",
        "        return self.encode(src, src_mask, adj_matrix, distances_matrix, edges_att)\n",
        "    \n",
        "    def encode(self, src, src_mask, adj_matrix, distances_matrix, edges_att):\n",
        "        return self.encoder(self.src_embed(src), src_mask, adj_matrix, distances_matrix, edges_att)\n",
        "    \n",
        "    def predict(self, out, out_mask):\n",
        "        return (out)\n",
        "    \n",
        "    \n",
        "class Generator(nn.Module):\n",
        "    \"Define standard linear + softmax generation step.\"\n",
        "    def __init__(self, d_model, aggregation_type='mean', n_output=1, n_layers=1, \n",
        "                 leaky_relu_slope=0.01, dropout=0.0, scale_norm=False):\n",
        "        super(Generator, self).__init__()\n",
        "        if n_layers == 1:\n",
        "            self.proj = nn.Linear(d_model, n_output)\n",
        "        else:\n",
        "            self.proj = []\n",
        "            for i in range(n_layers-1):\n",
        "                self.proj.append(nn.Linear(d_model, d_model))\n",
        "                self.proj.append(nn.LeakyReLU(leaky_relu_slope))\n",
        "                self.proj.append(ScaleNorm(d_model) if scale_norm else LayerNorm(d_model))\n",
        "                self.proj.append(nn.Dropout(dropout))\n",
        "            self.proj.append(nn.Linear(d_model, n_output))\n",
        "            self.proj = torch.nn.Sequential(*self.proj)\n",
        "        self.aggregation_type = aggregation_type\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "        mask = mask.unsqueeze(-1).float()\n",
        "        out_masked = x * mask\n",
        "        if self.aggregation_type == 'mean':\n",
        "            out_sum = out_masked.sum(dim=1)\n",
        "            mask_sum = mask.sum(dim=(1))\n",
        "            out_avg_pooling = out_sum / mask_sum\n",
        "        elif self.aggregation_type == 'sum':\n",
        "            out_sum = out_masked.sum(dim=1)\n",
        "            out_avg_pooling = out_sum\n",
        "        elif self.aggregation_type == 'dummy_node':\n",
        "            out_avg_pooling = out_masked[:,0]\n",
        "        projected = self.proj(out_avg_pooling)\n",
        "        return projected\n",
        "    \n",
        "    \n",
        "class PositionGenerator(nn.Module):\n",
        "    \"Define standard linear + softmax generation step.\"\n",
        "    def __init__(self, d_model):\n",
        "        super(PositionGenerator, self).__init__()\n",
        "        self.norm = LayerNorm(d_model)\n",
        "        self.proj = nn.Linear(d_model, 3)\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "        mask = mask.unsqueeze(-1).float()\n",
        "        out_masked = self.norm(x) * mask\n",
        "        projected = self.proj(out_masked)\n",
        "        return projected\n",
        "    \n",
        "\n",
        "### Encoder\n",
        "\n",
        "def clones(module, N):\n",
        "    \"Produce N identical layers.\"\n",
        "    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])\n",
        "\n",
        "\n",
        "class Encoder(nn.Module):\n",
        "    \"Core encoder is a stack of N layers\"\n",
        "    def __init__(self, layer, N, scale_norm):\n",
        "        super(Encoder, self).__init__()\n",
        "        self.layers = clones(layer, N)\n",
        "        self.norm = ScaleNorm(layer.size) if scale_norm else LayerNorm(layer.size)\n",
        "        \n",
        "    def forward(self, x, mask, adj_matrix, distances_matrix, edges_att):\n",
        "        \"Pass the input (and mask) through each layer in turn.\"\n",
        "        for layer in self.layers:\n",
        "            x = layer(x, mask, adj_matrix, distances_matrix, edges_att)\n",
        "        return self.norm(x)\n",
        "\n",
        "    \n",
        "class LayerNorm(nn.Module):\n",
        "    \"Construct a layernorm module (See citation for details).\"\n",
        "    def __init__(self, features, eps=1e-6):\n",
        "        super(LayerNorm, self).__init__()\n",
        "        self.a_2 = nn.Parameter(torch.ones(features))\n",
        "        self.b_2 = nn.Parameter(torch.zeros(features))\n",
        "        self.eps = eps\n",
        "\n",
        "    def forward(self, x):\n",
        "        mean = x.mean(-1, keepdim=True)\n",
        "        std = x.std(-1, keepdim=True)\n",
        "        return self.a_2 * (x - mean) / (std + self.eps) + self.b_2\n",
        "    \n",
        "    \n",
        "class ScaleNorm(nn.Module):\n",
        "    \"\"\"ScaleNorm\"\"\"\n",
        "    \"All g’s in SCALE NORM are initialized to sqrt(d)\"\n",
        "    def __init__(self, scale, eps=1e-5):\n",
        "        super(ScaleNorm, self).__init__()\n",
        "        self.scale = nn.Parameter(torch.tensor(math.sqrt(scale)))\n",
        "        self.eps = eps\n",
        "        \n",
        "    def forward(self, x):\n",
        "        norm = self.scale / torch.norm(x, dim=-1, keepdim=True).clamp(min=self.eps)\n",
        "        return x * norm\n",
        "\n",
        "    \n",
        "class SublayerConnection(nn.Module):\n",
        "    \"\"\"\n",
        "    A residual connection followed by a layer norm.\n",
        "    Note for code simplicity the norm is first as opposed to last.\n",
        "    \"\"\"\n",
        "    def __init__(self, size, dropout, scale_norm, use_adapter):\n",
        "        super(SublayerConnection, self).__init__()\n",
        "        self.norm = ScaleNorm(size) if scale_norm else LayerNorm(size)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        self.use_adapter = use_adapter\n",
        "        self.adapter = Adapter(size, 8) if use_adapter else None\n",
        "\n",
        "    def forward(self, x, sublayer):\n",
        "        \"Apply residual connection to any sublayer with the same size.\"\n",
        "        if self.use_adapter:\n",
        "            return x + self.dropout(self.adapter(sublayer(self.norm(x))))\n",
        "        return x + self.dropout(sublayer(self.norm(x)))\n",
        "\n",
        "    \n",
        "class EncoderLayer(nn.Module):\n",
        "    \"Encoder is made up of self-attn and feed forward (defined below)\"\n",
        "    def __init__(self, size, self_attn, feed_forward, dropout, scale_norm, use_adapter):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        self.self_attn = self_attn\n",
        "        self.feed_forward = feed_forward\n",
        "        self.sublayer = clones(SublayerConnection(size, dropout, scale_norm, use_adapter), 2)\n",
        "        self.size = size\n",
        "\n",
        "    def forward(self, x, mask, adj_matrix, distances_matrix, edges_att):\n",
        "        \"Follow Figure 1 (left) for connections.\"\n",
        "        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, adj_matrix, distances_matrix, edges_att, mask))\n",
        "        return self.sublayer[1](x, self.feed_forward)\n",
        "\n",
        "    \n",
        "### Attention           \n",
        "\n",
        "class EdgeFeaturesLayer(nn.Module):\n",
        "    def __init__(self, d_model, d_edge, h, dropout):\n",
        "        super(EdgeFeaturesLayer, self).__init__()\n",
        "        assert d_model % h == 0\n",
        "        d_k = d_model // h\n",
        "        self.linear = nn.Linear(d_edge, 1, bias=False)\n",
        "        with torch.no_grad():\n",
        "            self.linear.weight.fill_(0.25)\n",
        "\n",
        "    def forward(self, x):\n",
        "        p_edge = x.permute(0, 2, 3, 1)\n",
        "        p_edge = self.linear(p_edge).permute(0, 3, 1, 2)\n",
        "        return torch.relu(p_edge)\n",
        "    \n",
        "\n",
        "def attention(query, key, value, adj_matrix, distances_matrix, edges_att,\n",
        "              mask=None, dropout=None, \n",
        "              lambdas=(0.3, 0.3, 0.4), trainable_lambda=False,\n",
        "              distance_matrix_kernel=None, use_edge_features=False, control_edges=False,\n",
        "              eps=1e-6, inf=1e12):\n",
        "    \"Compute 'Scaled Dot Product Attention'\"\n",
        "    d_k = query.size(-1)\n",
        "    scores = torch.matmul(query, key.transpose(-2, -1)) \\\n",
        "             / math.sqrt(d_k)\n",
        "    if mask is not None:\n",
        "        scores = scores.masked_fill(mask.unsqueeze(1).repeat(1, query.shape[1], query.shape[2], 1) == 0, -inf)\n",
        "    p_attn = F.softmax(scores, dim = -1)\n",
        "\n",
        "    if use_edge_features:\n",
        "        adj_matrix = edges_att.view(adj_matrix.shape)\n",
        "\n",
        "    # Prepare adjacency matrix\n",
        "    adj_matrix = adj_matrix / (adj_matrix.sum(dim=-1).unsqueeze(2) + eps)\n",
        "    adj_matrix = adj_matrix.unsqueeze(1).repeat(1, query.shape[1], 1, 1)\n",
        "    p_adj = adj_matrix\n",
        "    \n",
        "    p_dist = distances_matrix\n",
        "    \n",
        "    if trainable_lambda:\n",
        "        softmax_attention, softmax_distance, softmax_adjacency = lambdas.cuda()\n",
        "        p_weighted = softmax_attention * p_attn + softmax_distance * p_dist + softmax_adjacency * p_adj\n",
        "    else:\n",
        "        lambda_attention, lambda_distance, lambda_adjacency = lambdas\n",
        "        p_weighted = lambda_attention * p_attn + lambda_distance * p_dist + lambda_adjacency * p_adj\n",
        "    \n",
        "    if dropout is not None:\n",
        "        p_weighted = dropout(p_weighted)\n",
        "\n",
        "    atoms_featrues = torch.matmul(p_weighted, value)     \n",
        "    return atoms_featrues, p_weighted, p_attn\n",
        "\n",
        "\n",
        "class MultiHeadedAttention(nn.Module):\n",
        "    def __init__(self, h, d_model, dropout=0.1, lambda_attention=0.3, lambda_distance=0.3, trainable_lambda=False, \n",
        "                 distance_matrix_kernel='softmax', use_edge_features=False, control_edges=False, integrated_distances=False):\n",
        "        \"Take in model size and number of heads.\"\n",
        "        super(MultiHeadedAttention, self).__init__()\n",
        "        assert d_model % h == 0\n",
        "        # We assume d_v always equals d_k\n",
        "        self.d_k = d_model // h\n",
        "        self.h = h\n",
        "        self.trainable_lambda = trainable_lambda\n",
        "        if trainable_lambda:\n",
        "            lambda_adjacency = 1. - lambda_attention - lambda_distance\n",
        "            lambdas_tensor = torch.tensor([lambda_attention, lambda_distance, lambda_adjacency], requires_grad=True)\n",
        "            self.lambdas = torch.nn.Parameter(lambdas_tensor)\n",
        "        else:\n",
        "            lambda_adjacency = 1. - lambda_attention - lambda_distance\n",
        "            self.lambdas = (lambda_attention, lambda_distance, lambda_adjacency)\n",
        "            \n",
        "        self.linears = clones(nn.Linear(d_model, d_model), 4)\n",
        "        self.attn = None\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "        if distance_matrix_kernel == 'softmax':\n",
        "            self.distance_matrix_kernel = lambda x: F.softmax(-x, dim = -1)\n",
        "        elif distance_matrix_kernel == 'exp':\n",
        "            self.distance_matrix_kernel = lambda x: torch.exp(-x)\n",
        "        self.integrated_distances = integrated_distances\n",
        "        self.use_edge_features = use_edge_features\n",
        "        self.control_edges = control_edges\n",
        "        if use_edge_features:\n",
        "            d_edge = 11 if not integrated_distances else 12\n",
        "            self.edges_feature_layer = EdgeFeaturesLayer(d_model, d_edge, h, dropout)\n",
        "        \n",
        "    def forward(self, query, key, value, adj_matrix, distances_matrix, edges_att, mask=None):\n",
        "        \"Implements Figure 2\"\n",
        "        if mask is not None:\n",
        "            # Same mask applied to all h heads.\n",
        "            mask = mask.unsqueeze(1)\n",
        "        nbatches = query.size(0)\n",
        "        \n",
        "        # 1) Do all the linear projections in batch from d_model => h x d_k \n",
        "        query, key, value = \\\n",
        "            [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n",
        "             for l, x in zip(self.linears, (query, key, value))]\n",
        "        \n",
        "        # Prepare distances matrix\n",
        "        distances_matrix = distances_matrix.masked_fill(mask.repeat(1, mask.shape[-1], 1) == 0, np.inf)\n",
        "        distances_matrix = self.distance_matrix_kernel(distances_matrix)\n",
        "        p_dist = distances_matrix.unsqueeze(1).repeat(1, query.shape[1], 1, 1)\n",
        "\n",
        "        if self.use_edge_features:\n",
        "            if self.integrated_distances:\n",
        "                edges_att = torch.cat((edges_att, distances_matrix.unsqueeze(1)), dim=1)\n",
        "            edges_att = self.edges_feature_layer(edges_att)\n",
        "        \n",
        "        # 2) Apply attention on all the projected vectors in batch. \n",
        "        x, self.attn, self.self_attn = attention(query, key, value, adj_matrix, \n",
        "                                                 p_dist, edges_att,\n",
        "                                                 mask=mask, dropout=self.dropout,\n",
        "                                                 lambdas=self.lambdas,\n",
        "                                                 trainable_lambda=self.trainable_lambda,\n",
        "                                                 distance_matrix_kernel=self.distance_matrix_kernel,\n",
        "                                                 use_edge_features=self.use_edge_features,\n",
        "                                                 control_edges=self.control_edges)\n",
        "        \n",
        "        # 3) \"Concat\" using a view and apply a final linear. \n",
        "        x = x.transpose(1, 2).contiguous() \\\n",
        "             .view(nbatches, -1, self.h * self.d_k)\n",
        "        return self.linears[-1](x)\n",
        "\n",
        "\n",
        "### Conv 1x1 aka Positionwise feed forward\n",
        "\n",
        "class PositionwiseFeedForward(nn.Module):\n",
        "    \"Implements FFN equation.\"\n",
        "    def __init__(self, d_model, N_dense, dropout=0.1, leaky_relu_slope=0.0, dense_output_nonlinearity='relu'):\n",
        "        super(PositionwiseFeedForward, self).__init__()\n",
        "        self.N_dense = N_dense\n",
        "        self.linears = clones(nn.Linear(d_model, d_model), N_dense)\n",
        "        self.dropout = clones(nn.Dropout(dropout), N_dense)\n",
        "        self.leaky_relu_slope = leaky_relu_slope\n",
        "        if dense_output_nonlinearity == 'relu':\n",
        "            self.dense_output_nonlinearity = lambda x: F.leaky_relu(x, negative_slope=self.leaky_relu_slope)\n",
        "        elif dense_output_nonlinearity == 'tanh':\n",
        "            self.tanh = torch.nn.Tanh()\n",
        "            self.dense_output_nonlinearity = lambda x: self.tanh(x)\n",
        "        elif dense_output_nonlinearity == 'none':\n",
        "            self.dense_output_nonlinearity = lambda x: x\n",
        "            \n",
        "\n",
        "    def forward(self, x):\n",
        "        if self.N_dense == 0:\n",
        "            return x\n",
        "        \n",
        "        for i in range(len(self.linears)-1):\n",
        "            x = self.dropout[i](F.leaky_relu(self.linears[i](x), negative_slope=self.leaky_relu_slope))\n",
        "            \n",
        "        return self.dropout[-1](self.dense_output_nonlinearity(self.linears[-1](x)))\n",
        "\n",
        "    \n",
        "## Embeddings\n",
        "\n",
        "class Embeddings(nn.Module):\n",
        "    def __init__(self, d_model, d_atom, dropout):\n",
        "        super(Embeddings, self).__init__()\n",
        "        self.lut = nn.Linear(d_atom, d_model)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.dropout(self.lut(x))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jEwD7AZww_9g"
      },
      "source": [
        "from en import make_models"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MWg5jRzqvq2c",
        "outputId": "15cf56d7-d161-4caa-b65f-7c6614d43005",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "\n",
        "d_atom = X[0][0].shape[1]  # It depends on the used featurization.\n",
        "\n",
        "model_params = {\n",
        "    'd_atom': d_atom,\n",
        "    'd_model': 1024,\n",
        "    'N': 8,\n",
        "    'h': 16,\n",
        "    'N_dense': 1,\n",
        "    'lambda_attention': 0.33, \n",
        "    'lambda_distance': 0.33,\n",
        "    'leaky_relu_slope': 0.1, \n",
        "    'dense_output_nonlinearity': 'relu', \n",
        "    'distance_matrix_kernel': 'exp', \n",
        "    'dropout': 0.0,\n",
        "    \n",
        "}\n",
        "\n",
        "model1 = make_models(**model_params)\n",
        "model1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GraphTransformer(\n",
              "  (encoder): Encoder(\n",
              "    (layers): ModuleList(\n",
              "      (0): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (norm): LayerNorm()\n",
              "  )\n",
              "  (src_embed): Embeddings(\n",
              "    (lut): Linear(in_features=28, out_features=1024, bias=True)\n",
              "    (dropout): Dropout(p=0.0, inplace=False)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TkNF0fKL0xy2",
        "outputId": "e34c8dc1-5dbb-4628-d810-880572b646c0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GraphTransformer(\n",
              "  (encoder): Encoder(\n",
              "    (layers): ModuleList(\n",
              "      (0): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (norm): LayerNorm()\n",
              "  )\n",
              "  (src_embed): Embeddings(\n",
              "    (lut): Linear(in_features=28, out_features=1024, bias=True)\n",
              "    (dropout): Dropout(p=0.0, inplace=False)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b9k_EtlZshZT"
      },
      "source": [
        "from transform import make_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VfLA_V4uhgy2",
        "outputId": "1fbf2e5a-78db-4ec1-f1e0-c6d52dfede44",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "make_model"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<function transformer.make_model>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FkEZMXwMxu7V",
        "outputId": "88381edd-d45f-43e6-ece1-17d276cc62b5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "\n",
        "d_atom = X[0][0].shape[1]  # It depends on the used featurization.\n",
        "\n",
        "model_params = {\n",
        "    'd_atom': d_atom,\n",
        "    'd_model': 1024,\n",
        "    'N': 8,\n",
        "    'h': 16,\n",
        "    'N_dense': 1,\n",
        "    'lambda_attention': 0.33, \n",
        "    'lambda_distance': 0.33,\n",
        "    'leaky_relu_slope': 0.1, \n",
        "    'dense_output_nonlinearity': 'relu', \n",
        "    'distance_matrix_kernel': 'exp', \n",
        "    'dropout': 0.0,\n",
        "    'aggregation_type': 'mean'\n",
        "}\n",
        "\n",
        "model = make_model(**model_params)\n",
        "model"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GraphTransformer(\n",
              "  (encoder): Encoder(\n",
              "    (layers): ModuleList(\n",
              "      (0): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (norm): LayerNorm()\n",
              "  )\n",
              "  (src_embed): Embeddings(\n",
              "    (lut): Linear(in_features=28, out_features=1024, bias=True)\n",
              "    (dropout): Dropout(p=0.0, inplace=False)\n",
              "  )\n",
              "  (generator): Generator(\n",
              "    (proj): Linear(in_features=1024, out_features=1, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3u77tdCrhH6g",
        "outputId": "411115c7-64fa-45d1-9c53-8b12764c898c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GraphTransformer(\n",
              "  (encoder): Encoder(\n",
              "    (layers): ModuleList(\n",
              "      (0): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (dropout): ModuleList(\n",
              "            (0): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.0, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (norm): LayerNorm()\n",
              "  )\n",
              "  (src_embed): Embeddings(\n",
              "    (lut): Linear(in_features=28, out_features=1024, bias=True)\n",
              "    (dropout): Dropout(p=0.0, inplace=False)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "klnq29WB4r6d"
      },
      "source": [
        "cp '/content/drive/My Drive/De NovoDrug/pretrained_weights.pt' /content/MAT/src"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z1MxRG9C5VEK"
      },
      "source": [
        "pretrained_name = 'pretrained_weights.pt'  # This file should be downloaded first (See README.md).\n",
        "pretrained_state_dict = torch.load(pretrained_name)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1gcxNIBo5bS6"
      },
      "source": [
        "model_state_dict = model1.state_dict()\n",
        "for name, param in pretrained_state_dict.items():\n",
        "    if 'generator' in name:\n",
        "         continue\n",
        "    if isinstance(param, torch.nn.Parameter):\n",
        "        param = param.data\n",
        "    model_state_dict[name].copy_(param)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5zuJxF-UtzsA"
      },
      "source": [
        "\n",
        "for batch in data_loader:\n",
        "\n",
        "    adjacency_matrix, node_features, distance_matrix, y = batch\n",
        "    batch_mask = torch.sum(torch.abs(node_features), dim=-1) != 0\n",
        "    output = mode1l(node_features, batch_mask, adjacency_matrix, distance_matrix, None)\n",
        "\n",
        "\n",
        "    \n",
        "x=output\n",
        "mask=batch_mask\n",
        "mask = mask.unsqueeze(-1).float()\n",
        "out_masked = x * mask\n",
        "out_sum = x.sum(dim=1)\n",
        "mask_sum = mask.sum(dim=(1))\n",
        "out_avg_pooling = out_sum / mask_sum\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0kDQ5inK3BQc"
      },
      "source": [
        "model_state_dict1 = model.state_dict()\n",
        "for name, param in pretrained_state_dict.items():\n",
        "    if 'generator' in name:\n",
        "         continue\n",
        "    if isinstance(param, torch.nn.Parameter):\n",
        "        param = param.data\n",
        "    model_state_dict[name].copy_(param)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2aZhaJ-rtpsU"
      },
      "source": [
        "\n",
        "for batch in data_loader:\n",
        "\n",
        "    adjacency_matrix, node_features, distance_matrix, y = batch\n",
        "    batch_mask = torch.sum(torch.abs(node_features), dim=-1) != 0\n",
        "    output = mode1l(node_features, batch_mask, adjacency_matrix, distance_matrix, None)\n",
        "\n",
        "\n",
        "    \n",
        "x=output\n",
        "mask=batch_mask\n",
        "mask = mask.unsqueeze(-1).float()\n",
        "out_masked = x * mask\n",
        "out_sum = x.sum(dim=1)\n",
        "mask_sum = mask.sum(dim=(1))\n",
        "out_avg_pooling = out_sum / mask_sum\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4oUOEuD-iIW6",
        "outputId": "b45bfa80-bd2d-42ad-84c5-77e5399c1a53",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model_state_dict1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "OrderedDict([('encoder.layers.0.self_attn.linears.0.weight',\n",
              "              tensor([[-0.0365,  0.0012, -0.0536,  ..., -0.0285,  0.0436, -0.0384],\n",
              "                      [ 0.0431,  0.0158,  0.0229,  ..., -0.0279,  0.0492, -0.0053],\n",
              "                      [-0.0359,  0.0122,  0.0405,  ..., -0.0511, -0.0131, -0.0218],\n",
              "                      ...,\n",
              "                      [ 0.0154,  0.0473,  0.0137,  ..., -0.0120,  0.0511, -0.0033],\n",
              "                      [-0.0473, -0.0241,  0.0178,  ...,  0.0360,  0.0265,  0.0203],\n",
              "                      [ 0.0168,  0.0155, -0.0262,  ..., -0.0123, -0.0207,  0.0021]])),\n",
              "             ('encoder.layers.0.self_attn.linears.0.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.0.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0387, -0.0216,  0.0536,  ..., -0.0044, -0.0305, -0.0069],\n",
              "                      [-0.0024, -0.0387, -0.0299,  ..., -0.0106, -0.0404, -0.0267],\n",
              "                      [-0.0120,  0.0314, -0.0338,  ..., -0.0148,  0.0024,  0.0168],\n",
              "                      ...,\n",
              "                      [-0.0485,  0.0446,  0.0167,  ...,  0.0526,  0.0255, -0.0314],\n",
              "                      [-0.0387,  0.0065,  0.0411,  ...,  0.0310,  0.0308,  0.0263],\n",
              "                      [ 0.0168, -0.0112,  0.0133,  ..., -0.0499,  0.0534,  0.0043]])),\n",
              "             ('encoder.layers.0.self_attn.linears.1.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.0.self_attn.linears.2.weight',\n",
              "              tensor([[ 0.0183,  0.0079,  0.0446,  ...,  0.0390,  0.0194,  0.0405],\n",
              "                      [-0.0158,  0.0122,  0.0205,  ..., -0.0414,  0.0258,  0.0283],\n",
              "                      [ 0.0356, -0.0247,  0.0171,  ...,  0.0251, -0.0459, -0.0533],\n",
              "                      ...,\n",
              "                      [-0.0245, -0.0454, -0.0250,  ..., -0.0291,  0.0003,  0.0005],\n",
              "                      [ 0.0372,  0.0420, -0.0124,  ..., -0.0253, -0.0451, -0.0462],\n",
              "                      [ 0.0177,  0.0270, -0.0131,  ...,  0.0374,  0.0035,  0.0161]])),\n",
              "             ('encoder.layers.0.self_attn.linears.2.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.0.self_attn.linears.3.weight',\n",
              "              tensor([[-0.0218,  0.0179,  0.0034,  ..., -0.0052,  0.0285,  0.0078],\n",
              "                      [-0.0005,  0.0532,  0.0170,  ..., -0.0451, -0.0033,  0.0360],\n",
              "                      [-0.0048, -0.0431, -0.0528,  ..., -0.0361,  0.0195, -0.0210],\n",
              "                      ...,\n",
              "                      [-0.0275, -0.0525,  0.0008,  ..., -0.0104, -0.0288,  0.0299],\n",
              "                      [-0.0409,  0.0069, -0.0460,  ...,  0.0089,  0.0196, -0.0479],\n",
              "                      [-0.0230, -0.0402,  0.0042,  ...,  0.0299, -0.0381, -0.0013]])),\n",
              "             ('encoder.layers.0.self_attn.linears.3.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.0.feed_forward.linears.0.weight',\n",
              "              tensor([[ 5.8531e-03,  5.3609e-02, -3.4259e-02,  ..., -2.8834e-02,\n",
              "                        6.2812e-03, -1.8068e-02],\n",
              "                      [-7.9402e-03, -2.5620e-02,  1.8155e-03,  ..., -4.9443e-02,\n",
              "                       -1.0344e-02,  1.7874e-02],\n",
              "                      [-5.1762e-02,  5.0068e-02,  7.1309e-04,  ..., -3.9254e-02,\n",
              "                       -2.8742e-02,  1.8249e-02],\n",
              "                      ...,\n",
              "                      [ 1.5676e-02, -1.6547e-02, -2.4157e-02,  ...,  4.6800e-02,\n",
              "                        4.3624e-02, -3.1296e-02],\n",
              "                      [ 1.9453e-03,  3.2791e-05, -4.4853e-02,  ..., -2.9278e-02,\n",
              "                        1.6732e-02,  2.9099e-02],\n",
              "                      [ 1.0516e-02, -1.1883e-03, -3.6771e-03,  ..., -9.4753e-03,\n",
              "                        3.1245e-02, -1.2173e-02]])),\n",
              "             ('encoder.layers.0.feed_forward.linears.0.bias',\n",
              "              tensor([-0.0011,  0.0296,  0.0139,  ..., -0.0297,  0.0307,  0.0093])),\n",
              "             ('encoder.layers.0.sublayer.0.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.0.sublayer.0.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.0.sublayer.1.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.0.sublayer.1.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.1.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0105,  0.0475, -0.0308,  ...,  0.0531, -0.0076, -0.0013],\n",
              "                      [-0.0088, -0.0038, -0.0428,  ...,  0.0010,  0.0040, -0.0347],\n",
              "                      [-0.0481,  0.0294,  0.0492,  ..., -0.0030, -0.0496,  0.0446],\n",
              "                      ...,\n",
              "                      [-0.0419,  0.0110, -0.0147,  ..., -0.0342, -0.0258,  0.0370],\n",
              "                      [ 0.0358, -0.0451, -0.0492,  ..., -0.0205,  0.0264, -0.0514],\n",
              "                      [ 0.0111, -0.0444, -0.0260,  ...,  0.0268, -0.0235,  0.0276]])),\n",
              "             ('encoder.layers.1.self_attn.linears.0.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.1.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0476, -0.0367,  0.0031,  ..., -0.0258,  0.0212, -0.0317],\n",
              "                      [-0.0033,  0.0502,  0.0157,  ...,  0.0538, -0.0085, -0.0470],\n",
              "                      [-0.0386,  0.0299, -0.0302,  ...,  0.0499,  0.0256, -0.0084],\n",
              "                      ...,\n",
              "                      [ 0.0519,  0.0298,  0.0454,  ..., -0.0417,  0.0176,  0.0461],\n",
              "                      [ 0.0052, -0.0038, -0.0081,  ..., -0.0426, -0.0009, -0.0454],\n",
              "                      [ 0.0063, -0.0353,  0.0221,  ..., -0.0249, -0.0491, -0.0014]])),\n",
              "             ('encoder.layers.1.self_attn.linears.1.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.1.self_attn.linears.2.weight',\n",
              "              tensor([[ 0.0332, -0.0456, -0.0351,  ...,  0.0215, -0.0013, -0.0285],\n",
              "                      [-0.0087,  0.0381, -0.0226,  ..., -0.0237, -0.0009, -0.0430],\n",
              "                      [ 0.0352,  0.0339,  0.0375,  ...,  0.0061,  0.0044, -0.0210],\n",
              "                      ...,\n",
              "                      [ 0.0155, -0.0057,  0.0510,  ...,  0.0424,  0.0201,  0.0105],\n",
              "                      [-0.0227, -0.0049,  0.0097,  ...,  0.0407,  0.0074,  0.0288],\n",
              "                      [ 0.0065, -0.0383,  0.0012,  ..., -0.0178,  0.0114, -0.0480]])),\n",
              "             ('encoder.layers.1.self_attn.linears.2.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.1.self_attn.linears.3.weight',\n",
              "              tensor([[-8.7461e-03, -9.3137e-03, -2.8949e-02,  ..., -3.8727e-03,\n",
              "                        4.7585e-02,  3.2513e-03],\n",
              "                      [ 4.3154e-03,  1.2925e-03, -2.2877e-03,  ...,  6.7147e-04,\n",
              "                        4.2565e-02, -3.6499e-02],\n",
              "                      [ 3.8080e-02,  4.2180e-02,  1.8488e-02,  ..., -4.4647e-02,\n",
              "                       -4.7238e-02, -3.9057e-02],\n",
              "                      ...,\n",
              "                      [-1.9141e-02, -1.8363e-02,  1.7570e-02,  ...,  2.6853e-02,\n",
              "                       -4.6684e-02,  1.6780e-02],\n",
              "                      [ 1.5950e-02,  3.9201e-02, -1.9545e-02,  ...,  3.3267e-02,\n",
              "                        3.4282e-02, -5.2423e-02],\n",
              "                      [ 5.1768e-02, -2.4676e-02,  3.2379e-02,  ...,  5.1981e-02,\n",
              "                        8.5714e-05, -1.1322e-02]])),\n",
              "             ('encoder.layers.1.self_attn.linears.3.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.1.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0258,  0.0210, -0.0452,  ..., -0.0506, -0.0219, -0.0374],\n",
              "                      [ 0.0509,  0.0191, -0.0249,  ...,  0.0524,  0.0344, -0.0432],\n",
              "                      [ 0.0503, -0.0403,  0.0181,  ...,  0.0342,  0.0265,  0.0199],\n",
              "                      ...,\n",
              "                      [ 0.0494,  0.0072, -0.0404,  ..., -0.0449, -0.0130,  0.0054],\n",
              "                      [ 0.0518,  0.0451, -0.0224,  ..., -0.0080, -0.0333,  0.0315],\n",
              "                      [ 0.0031, -0.0510,  0.0293,  ...,  0.0201,  0.0429,  0.0064]])),\n",
              "             ('encoder.layers.1.feed_forward.linears.0.bias',\n",
              "              tensor([-0.0011,  0.0296,  0.0139,  ..., -0.0297,  0.0307,  0.0093])),\n",
              "             ('encoder.layers.1.sublayer.0.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.1.sublayer.0.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.1.sublayer.1.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.1.sublayer.1.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.2.self_attn.linears.0.weight',\n",
              "              tensor([[-0.0223, -0.0043,  0.0385,  ..., -0.0057,  0.0268,  0.0047],\n",
              "                      [ 0.0191,  0.0273,  0.0249,  ...,  0.0180, -0.0018, -0.0455],\n",
              "                      [-0.0131,  0.0385, -0.0228,  ..., -0.0081, -0.0289, -0.0520],\n",
              "                      ...,\n",
              "                      [-0.0506, -0.0119,  0.0406,  ..., -0.0452, -0.0139, -0.0128],\n",
              "                      [-0.0211,  0.0538, -0.0533,  ...,  0.0421,  0.0055,  0.0462],\n",
              "                      [ 0.0118, -0.0103, -0.0351,  ..., -0.0487, -0.0523, -0.0140]])),\n",
              "             ('encoder.layers.2.self_attn.linears.0.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.2.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0061,  0.0368,  0.0043,  ...,  0.0253,  0.0186,  0.0387],\n",
              "                      [ 0.0270,  0.0363,  0.0237,  ..., -0.0438, -0.0480, -0.0356],\n",
              "                      [-0.0074, -0.0357,  0.0399,  ...,  0.0310,  0.0104, -0.0468],\n",
              "                      ...,\n",
              "                      [ 0.0237,  0.0324, -0.0222,  ..., -0.0059,  0.0335,  0.0198],\n",
              "                      [-0.0399, -0.0011,  0.0190,  ...,  0.0394,  0.0028,  0.0520],\n",
              "                      [ 0.0191, -0.0039, -0.0338,  ..., -0.0215,  0.0454, -0.0264]])),\n",
              "             ('encoder.layers.2.self_attn.linears.1.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.2.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0500, -0.0230,  0.0405,  ...,  0.0237,  0.0206,  0.0266],\n",
              "                      [ 0.0129,  0.0198, -0.0391,  ...,  0.0070, -0.0152,  0.0209],\n",
              "                      [-0.0112, -0.0515, -0.0106,  ...,  0.0079,  0.0352,  0.0016],\n",
              "                      ...,\n",
              "                      [-0.0154, -0.0489, -0.0459,  ..., -0.0224, -0.0464, -0.0231],\n",
              "                      [-0.0434, -0.0513, -0.0121,  ..., -0.0497,  0.0167, -0.0325],\n",
              "                      [-0.0130,  0.0287,  0.0008,  ...,  0.0285, -0.0106, -0.0363]])),\n",
              "             ('encoder.layers.2.self_attn.linears.2.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.2.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0090, -0.0397,  0.0062,  ..., -0.0133,  0.0115,  0.0376],\n",
              "                      [ 0.0155,  0.0518,  0.0285,  ..., -0.0187, -0.0514, -0.0464],\n",
              "                      [-0.0121,  0.0077,  0.0207,  ..., -0.0296, -0.0193, -0.0501],\n",
              "                      ...,\n",
              "                      [-0.0152, -0.0186, -0.0383,  ...,  0.0148, -0.0360,  0.0349],\n",
              "                      [-0.0269,  0.0127, -0.0404,  ...,  0.0527,  0.0442,  0.0458],\n",
              "                      [-0.0294, -0.0296,  0.0296,  ..., -0.0289,  0.0262,  0.0314]])),\n",
              "             ('encoder.layers.2.self_attn.linears.3.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.2.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0248,  0.0002,  0.0495,  ...,  0.0378,  0.0113, -0.0094],\n",
              "                      [ 0.0288, -0.0407,  0.0524,  ..., -0.0178,  0.0159, -0.0235],\n",
              "                      [ 0.0368,  0.0131, -0.0350,  ..., -0.0378,  0.0079, -0.0437],\n",
              "                      ...,\n",
              "                      [ 0.0076, -0.0383,  0.0223,  ..., -0.0044,  0.0334,  0.0391],\n",
              "                      [-0.0517, -0.0300,  0.0381,  ...,  0.0368,  0.0284, -0.0023],\n",
              "                      [-0.0334,  0.0196, -0.0293,  ..., -0.0124,  0.0445,  0.0308]])),\n",
              "             ('encoder.layers.2.feed_forward.linears.0.bias',\n",
              "              tensor([-0.0011,  0.0296,  0.0139,  ..., -0.0297,  0.0307,  0.0093])),\n",
              "             ('encoder.layers.2.sublayer.0.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.2.sublayer.0.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.2.sublayer.1.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.2.sublayer.1.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.3.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0490, -0.0511, -0.0228,  ..., -0.0245, -0.0434, -0.0037],\n",
              "                      [-0.0174, -0.0055, -0.0398,  ..., -0.0530, -0.0018, -0.0125],\n",
              "                      [ 0.0452, -0.0441,  0.0053,  ...,  0.0071, -0.0086, -0.0347],\n",
              "                      ...,\n",
              "                      [-0.0260,  0.0311, -0.0277,  ...,  0.0356, -0.0364, -0.0401],\n",
              "                      [-0.0247,  0.0400, -0.0023,  ...,  0.0489,  0.0317,  0.0220],\n",
              "                      [ 0.0033,  0.0019,  0.0530,  ...,  0.0342, -0.0127, -0.0054]])),\n",
              "             ('encoder.layers.3.self_attn.linears.0.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.3.self_attn.linears.1.weight',\n",
              "              tensor([[ 0.0227,  0.0015, -0.0155,  ..., -0.0143,  0.0065, -0.0458],\n",
              "                      [-0.0467,  0.0088, -0.0336,  ..., -0.0094,  0.0038, -0.0106],\n",
              "                      [ 0.0382, -0.0401,  0.0264,  ...,  0.0048, -0.0379, -0.0212],\n",
              "                      ...,\n",
              "                      [ 0.0279, -0.0162,  0.0445,  ...,  0.0522,  0.0435, -0.0420],\n",
              "                      [ 0.0173,  0.0308, -0.0007,  ...,  0.0419,  0.0341, -0.0367],\n",
              "                      [-0.0182,  0.0257, -0.0327,  ..., -0.0385, -0.0509,  0.0387]])),\n",
              "             ('encoder.layers.3.self_attn.linears.1.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.3.self_attn.linears.2.weight',\n",
              "              tensor([[ 0.0114,  0.0268,  0.0491,  ..., -0.0519, -0.0266, -0.0345],\n",
              "                      [-0.0448,  0.0153, -0.0019,  ...,  0.0183, -0.0286,  0.0311],\n",
              "                      [ 0.0249, -0.0305,  0.0209,  ...,  0.0058, -0.0110, -0.0179],\n",
              "                      ...,\n",
              "                      [-0.0075,  0.0525,  0.0300,  ...,  0.0106, -0.0218, -0.0191],\n",
              "                      [-0.0440, -0.0239, -0.0184,  ...,  0.0440,  0.0286, -0.0379],\n",
              "                      [-0.0040,  0.0315,  0.0146,  ...,  0.0083,  0.0304,  0.0260]])),\n",
              "             ('encoder.layers.3.self_attn.linears.2.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.3.self_attn.linears.3.weight',\n",
              "              tensor([[-0.0218, -0.0154, -0.0040,  ..., -0.0445, -0.0248, -0.0393],\n",
              "                      [-0.0480, -0.0529, -0.0151,  ...,  0.0344,  0.0017, -0.0022],\n",
              "                      [ 0.0061,  0.0006, -0.0121,  ...,  0.0531,  0.0470, -0.0047],\n",
              "                      ...,\n",
              "                      [ 0.0060,  0.0205,  0.0464,  ..., -0.0452, -0.0064,  0.0439],\n",
              "                      [-0.0134,  0.0429, -0.0413,  ..., -0.0126, -0.0389,  0.0362],\n",
              "                      [ 0.0126,  0.0044,  0.0188,  ..., -0.0360,  0.0363, -0.0238]])),\n",
              "             ('encoder.layers.3.self_attn.linears.3.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.3.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0322,  0.0467, -0.0422,  ...,  0.0199,  0.0041,  0.0298],\n",
              "                      [ 0.0397, -0.0541, -0.0312,  ...,  0.0237, -0.0447,  0.0270],\n",
              "                      [ 0.0468, -0.0436, -0.0049,  ..., -0.0367,  0.0537,  0.0065],\n",
              "                      ...,\n",
              "                      [ 0.0029, -0.0315, -0.0513,  ..., -0.0463,  0.0239,  0.0532],\n",
              "                      [-0.0520, -0.0427,  0.0133,  ..., -0.0531,  0.0540, -0.0023],\n",
              "                      [ 0.0052,  0.0209, -0.0112,  ..., -0.0175, -0.0501, -0.0129]])),\n",
              "             ('encoder.layers.3.feed_forward.linears.0.bias',\n",
              "              tensor([-0.0011,  0.0296,  0.0139,  ..., -0.0297,  0.0307,  0.0093])),\n",
              "             ('encoder.layers.3.sublayer.0.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.3.sublayer.0.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.3.sublayer.1.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.3.sublayer.1.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.4.self_attn.linears.0.weight',\n",
              "              tensor([[-0.0293, -0.0220, -0.0089,  ...,  0.0363,  0.0419,  0.0422],\n",
              "                      [ 0.0093, -0.0328, -0.0147,  ..., -0.0056, -0.0125, -0.0250],\n",
              "                      [-0.0202,  0.0496,  0.0077,  ..., -0.0465,  0.0135, -0.0223],\n",
              "                      ...,\n",
              "                      [-0.0323,  0.0238, -0.0204,  ..., -0.0321,  0.0104, -0.0033],\n",
              "                      [ 0.0067,  0.0343, -0.0278,  ...,  0.0028,  0.0304, -0.0363],\n",
              "                      [ 0.0093,  0.0353,  0.0381,  ...,  0.0151,  0.0296,  0.0434]])),\n",
              "             ('encoder.layers.4.self_attn.linears.0.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.4.self_attn.linears.1.weight',\n",
              "              tensor([[ 0.0364, -0.0043,  0.0106,  ...,  0.0296, -0.0373, -0.0190],\n",
              "                      [-0.0155,  0.0290, -0.0269,  ..., -0.0370,  0.0324, -0.0040],\n",
              "                      [-0.0040, -0.0310, -0.0530,  ..., -0.0022,  0.0103, -0.0064],\n",
              "                      ...,\n",
              "                      [ 0.0227, -0.0368, -0.0261,  ..., -0.0421,  0.0230, -0.0071],\n",
              "                      [ 0.0341,  0.0460, -0.0455,  ..., -0.0306, -0.0352, -0.0044],\n",
              "                      [ 0.0076,  0.0175, -0.0142,  ..., -0.0141, -0.0185,  0.0510]])),\n",
              "             ('encoder.layers.4.self_attn.linears.1.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.4.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0301, -0.0508,  0.0307,  ..., -0.0299,  0.0053,  0.0115],\n",
              "                      [-0.0107, -0.0183,  0.0468,  ...,  0.0344, -0.0457, -0.0315],\n",
              "                      [-0.0320, -0.0167,  0.0273,  ..., -0.0450, -0.0326, -0.0386],\n",
              "                      ...,\n",
              "                      [-0.0421,  0.0240, -0.0200,  ..., -0.0538, -0.0242, -0.0390],\n",
              "                      [-0.0480,  0.0421, -0.0388,  ...,  0.0412,  0.0105,  0.0292],\n",
              "                      [ 0.0471,  0.0047, -0.0126,  ..., -0.0185, -0.0372, -0.0225]])),\n",
              "             ('encoder.layers.4.self_attn.linears.2.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.4.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0395,  0.0215, -0.0508,  ..., -0.0190,  0.0378,  0.0193],\n",
              "                      [ 0.0126, -0.0110, -0.0342,  ..., -0.0272, -0.0226, -0.0385],\n",
              "                      [ 0.0099,  0.0114, -0.0254,  ..., -0.0139, -0.0484, -0.0363],\n",
              "                      ...,\n",
              "                      [ 0.0248,  0.0008,  0.0146,  ..., -0.0076, -0.0421, -0.0130],\n",
              "                      [-0.0404, -0.0226, -0.0411,  ...,  0.0265,  0.0175,  0.0425],\n",
              "                      [ 0.0104, -0.0497,  0.0306,  ...,  0.0132, -0.0464, -0.0410]])),\n",
              "             ('encoder.layers.4.self_attn.linears.3.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.4.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0456, -0.0037,  0.0354,  ...,  0.0231,  0.0049, -0.0449],\n",
              "                      [-0.0104,  0.0518,  0.0207,  ..., -0.0038,  0.0032, -0.0009],\n",
              "                      [-0.0458, -0.0487, -0.0423,  ...,  0.0199,  0.0464, -0.0051],\n",
              "                      ...,\n",
              "                      [-0.0229,  0.0390,  0.0298,  ..., -0.0475,  0.0177, -0.0356],\n",
              "                      [ 0.0097,  0.0516,  0.0052,  ..., -0.0439,  0.0248,  0.0349],\n",
              "                      [ 0.0262,  0.0003,  0.0347,  ...,  0.0406,  0.0447,  0.0462]])),\n",
              "             ('encoder.layers.4.feed_forward.linears.0.bias',\n",
              "              tensor([-0.0011,  0.0296,  0.0139,  ..., -0.0297,  0.0307,  0.0093])),\n",
              "             ('encoder.layers.4.sublayer.0.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.4.sublayer.0.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.4.sublayer.1.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.4.sublayer.1.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.5.self_attn.linears.0.weight',\n",
              "              tensor([[ 2.3401e-02, -1.2612e-02,  3.5582e-02,  ...,  5.2675e-02,\n",
              "                        4.9436e-02, -4.3067e-02],\n",
              "                      [-7.0275e-03,  8.8664e-03,  4.9513e-02,  ..., -2.1499e-02,\n",
              "                       -4.0507e-02, -4.5043e-03],\n",
              "                      [ 4.4313e-02,  5.0974e-05, -4.1887e-02,  ...,  4.4259e-02,\n",
              "                       -1.5997e-02, -2.0517e-02],\n",
              "                      ...,\n",
              "                      [-2.1143e-02, -2.7143e-02,  2.6762e-02,  ...,  3.6301e-03,\n",
              "                       -5.0862e-02, -4.0953e-02],\n",
              "                      [-3.6736e-02,  1.2823e-02, -3.8976e-02,  ...,  3.4755e-02,\n",
              "                        3.7258e-02,  4.8843e-02],\n",
              "                      [ 5.3788e-02,  2.4752e-02, -2.2574e-02,  ..., -1.7574e-02,\n",
              "                       -1.2041e-02, -4.5464e-02]])),\n",
              "             ('encoder.layers.5.self_attn.linears.0.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.5.self_attn.linears.1.weight',\n",
              "              tensor([[ 2.2649e-02, -2.5930e-02, -2.4160e-02,  ..., -4.8536e-02,\n",
              "                       -1.8269e-02, -4.8704e-02],\n",
              "                      [ 2.1499e-02, -3.9915e-02, -7.2718e-05,  ...,  2.9585e-02,\n",
              "                        2.8742e-02,  4.0826e-03],\n",
              "                      [-4.2677e-02,  3.2255e-02, -3.2493e-03,  ...,  3.3900e-02,\n",
              "                        1.1216e-02, -2.7877e-02],\n",
              "                      ...,\n",
              "                      [-7.3773e-03, -3.2099e-02,  1.7313e-02,  ..., -1.5078e-02,\n",
              "                       -2.2612e-02,  2.0346e-02],\n",
              "                      [-4.8731e-02, -1.2474e-02, -7.1038e-03,  ...,  2.1355e-02,\n",
              "                        8.6952e-03,  3.6983e-02],\n",
              "                      [ 2.9558e-02, -2.2403e-02,  4.0756e-02,  ...,  1.7819e-03,\n",
              "                        5.0972e-02,  3.3017e-02]])),\n",
              "             ('encoder.layers.5.self_attn.linears.1.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.5.self_attn.linears.2.weight',\n",
              "              tensor([[ 0.0013,  0.0260,  0.0168,  ..., -0.0333, -0.0495,  0.0102],\n",
              "                      [-0.0278, -0.0096, -0.0128,  ...,  0.0342, -0.0530,  0.0430],\n",
              "                      [-0.0395,  0.0427,  0.0525,  ...,  0.0265, -0.0015,  0.0393],\n",
              "                      ...,\n",
              "                      [-0.0541,  0.0113,  0.0263,  ...,  0.0154,  0.0317,  0.0242],\n",
              "                      [ 0.0069,  0.0390, -0.0120,  ..., -0.0219,  0.0470,  0.0104],\n",
              "                      [-0.0443, -0.0022, -0.0505,  ...,  0.0289,  0.0123,  0.0010]])),\n",
              "             ('encoder.layers.5.self_attn.linears.2.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.5.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0034, -0.0219,  0.0322,  ..., -0.0304, -0.0236, -0.0233],\n",
              "                      [-0.0349, -0.0357, -0.0024,  ..., -0.0011, -0.0311,  0.0123],\n",
              "                      [-0.0529, -0.0421, -0.0285,  ...,  0.0075, -0.0266,  0.0490],\n",
              "                      ...,\n",
              "                      [ 0.0190, -0.0423, -0.0332,  ..., -0.0029, -0.0441,  0.0282],\n",
              "                      [-0.0272, -0.0077,  0.0331,  ..., -0.0401, -0.0492,  0.0172],\n",
              "                      [-0.0290, -0.0022,  0.0058,  ..., -0.0428,  0.0186,  0.0414]])),\n",
              "             ('encoder.layers.5.self_attn.linears.3.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.5.feed_forward.linears.0.weight',\n",
              "              tensor([[-3.7145e-02,  2.1992e-02,  4.0786e-02,  ...,  8.4020e-03,\n",
              "                       -3.9881e-02, -3.8020e-02],\n",
              "                      [-5.2612e-02,  8.6775e-03, -4.6725e-02,  ..., -1.0064e-02,\n",
              "                        2.5024e-02,  9.8110e-03],\n",
              "                      [ 4.7128e-02, -2.6751e-02, -4.2028e-03,  ..., -5.0504e-02,\n",
              "                       -4.7649e-02,  2.1569e-03],\n",
              "                      ...,\n",
              "                      [ 1.9651e-02, -3.0300e-05,  1.6192e-02,  ..., -1.4924e-02,\n",
              "                       -5.7386e-04, -4.4077e-02],\n",
              "                      [ 3.4636e-02,  2.7316e-03, -9.8917e-03,  ...,  4.0110e-02,\n",
              "                       -2.5848e-02,  4.6173e-02],\n",
              "                      [-4.0149e-03, -3.1407e-02, -2.5382e-02,  ...,  5.3335e-02,\n",
              "                       -1.4732e-02,  5.2451e-02]])),\n",
              "             ('encoder.layers.5.feed_forward.linears.0.bias',\n",
              "              tensor([-0.0011,  0.0296,  0.0139,  ..., -0.0297,  0.0307,  0.0093])),\n",
              "             ('encoder.layers.5.sublayer.0.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.5.sublayer.0.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.5.sublayer.1.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.5.sublayer.1.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.6.self_attn.linears.0.weight',\n",
              "              tensor([[-0.0368, -0.0513,  0.0439,  ...,  0.0229, -0.0277,  0.0456],\n",
              "                      [ 0.0444,  0.0444, -0.0196,  ..., -0.0168,  0.0381, -0.0168],\n",
              "                      [ 0.0367,  0.0065, -0.0239,  ..., -0.0364, -0.0423,  0.0329],\n",
              "                      ...,\n",
              "                      [-0.0436, -0.0404, -0.0320,  ..., -0.0448, -0.0263, -0.0165],\n",
              "                      [-0.0386, -0.0501, -0.0482,  ...,  0.0289,  0.0179,  0.0399],\n",
              "                      [-0.0321,  0.0172, -0.0184,  ..., -0.0310,  0.0478, -0.0274]])),\n",
              "             ('encoder.layers.6.self_attn.linears.0.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.6.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0401, -0.0379,  0.0386,  ...,  0.0371, -0.0481, -0.0537],\n",
              "                      [-0.0003,  0.0251,  0.0080,  ...,  0.0034, -0.0086, -0.0252],\n",
              "                      [-0.0130, -0.0169, -0.0029,  ...,  0.0451,  0.0007, -0.0272],\n",
              "                      ...,\n",
              "                      [-0.0435,  0.0294,  0.0112,  ..., -0.0315,  0.0022,  0.0401],\n",
              "                      [-0.0245,  0.0133, -0.0019,  ...,  0.0516,  0.0402,  0.0232],\n",
              "                      [ 0.0492,  0.0082, -0.0381,  ...,  0.0082,  0.0487, -0.0424]])),\n",
              "             ('encoder.layers.6.self_attn.linears.1.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.6.self_attn.linears.2.weight',\n",
              "              tensor([[ 0.0406,  0.0098,  0.0228,  ...,  0.0269,  0.0202, -0.0456],\n",
              "                      [ 0.0010,  0.0099, -0.0164,  ...,  0.0165, -0.0130, -0.0126],\n",
              "                      [ 0.0312,  0.0065,  0.0389,  ..., -0.0091,  0.0420, -0.0324],\n",
              "                      ...,\n",
              "                      [ 0.0290,  0.0088,  0.0113,  ...,  0.0083,  0.0165,  0.0199],\n",
              "                      [ 0.0222, -0.0206,  0.0066,  ...,  0.0117,  0.0526, -0.0159],\n",
              "                      [-0.0125, -0.0101, -0.0277,  ...,  0.0335,  0.0023,  0.0358]])),\n",
              "             ('encoder.layers.6.self_attn.linears.2.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.6.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0426, -0.0306,  0.0364,  ..., -0.0367,  0.0357, -0.0101],\n",
              "                      [ 0.0475,  0.0030, -0.0159,  ..., -0.0470,  0.0156,  0.0212],\n",
              "                      [-0.0530, -0.0173, -0.0275,  ...,  0.0483,  0.0519, -0.0180],\n",
              "                      ...,\n",
              "                      [-0.0513,  0.0465, -0.0453,  ...,  0.0071,  0.0515,  0.0259],\n",
              "                      [-0.0448, -0.0298, -0.0357,  ...,  0.0356, -0.0415, -0.0262],\n",
              "                      [ 0.0423,  0.0247, -0.0534,  ...,  0.0251, -0.0261,  0.0372]])),\n",
              "             ('encoder.layers.6.self_attn.linears.3.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.6.feed_forward.linears.0.weight',\n",
              "              tensor([[ 0.0301,  0.0466, -0.0326,  ..., -0.0380,  0.0458,  0.0277],\n",
              "                      [ 0.0245,  0.0475, -0.0322,  ...,  0.0211, -0.0221, -0.0391],\n",
              "                      [-0.0500, -0.0208,  0.0398,  ..., -0.0169, -0.0533,  0.0534],\n",
              "                      ...,\n",
              "                      [ 0.0005,  0.0040,  0.0290,  ..., -0.0153, -0.0031,  0.0140],\n",
              "                      [ 0.0398,  0.0174, -0.0490,  ..., -0.0295, -0.0355,  0.0057],\n",
              "                      [ 0.0505,  0.0323, -0.0025,  ...,  0.0413, -0.0198,  0.0428]])),\n",
              "             ('encoder.layers.6.feed_forward.linears.0.bias',\n",
              "              tensor([-0.0011,  0.0296,  0.0139,  ..., -0.0297,  0.0307,  0.0093])),\n",
              "             ('encoder.layers.6.sublayer.0.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.6.sublayer.0.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.6.sublayer.1.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.6.sublayer.1.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.7.self_attn.linears.0.weight',\n",
              "              tensor([[-0.0025, -0.0488, -0.0267,  ...,  0.0219, -0.0377, -0.0140],\n",
              "                      [ 0.0084, -0.0034, -0.0184,  ..., -0.0475, -0.0389, -0.0004],\n",
              "                      [-0.0007,  0.0089, -0.0324,  ...,  0.0325, -0.0063,  0.0293],\n",
              "                      ...,\n",
              "                      [ 0.0378, -0.0095,  0.0017,  ..., -0.0309,  0.0201, -0.0032],\n",
              "                      [ 0.0231,  0.0329, -0.0242,  ..., -0.0436,  0.0151,  0.0425],\n",
              "                      [ 0.0106, -0.0501,  0.0386,  ...,  0.0082, -0.0538,  0.0132]])),\n",
              "             ('encoder.layers.7.self_attn.linears.0.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.7.self_attn.linears.1.weight',\n",
              "              tensor([[ 0.0133, -0.0111, -0.0195,  ..., -0.0309,  0.0217,  0.0350],\n",
              "                      [-0.0214,  0.0320,  0.0493,  ...,  0.0357,  0.0226, -0.0122],\n",
              "                      [-0.0540, -0.0357, -0.0344,  ..., -0.0063,  0.0533, -0.0405],\n",
              "                      ...,\n",
              "                      [ 0.0142, -0.0211, -0.0324,  ..., -0.0393, -0.0165, -0.0346],\n",
              "                      [-0.0449,  0.0489, -0.0529,  ..., -0.0522,  0.0012,  0.0098],\n",
              "                      [ 0.0225, -0.0431,  0.0317,  ...,  0.0278,  0.0057,  0.0380]])),\n",
              "             ('encoder.layers.7.self_attn.linears.1.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.7.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0466,  0.0080, -0.0152,  ...,  0.0368, -0.0266,  0.0285],\n",
              "                      [-0.0370,  0.0262,  0.0456,  ...,  0.0539,  0.0029, -0.0262],\n",
              "                      [-0.0335,  0.0474, -0.0492,  ..., -0.0364, -0.0268, -0.0375],\n",
              "                      ...,\n",
              "                      [ 0.0086,  0.0147, -0.0527,  ...,  0.0484,  0.0044,  0.0163],\n",
              "                      [ 0.0230, -0.0229,  0.0525,  ...,  0.0119,  0.0257, -0.0103],\n",
              "                      [-0.0372, -0.0245,  0.0047,  ..., -0.0054, -0.0117,  0.0014]])),\n",
              "             ('encoder.layers.7.self_attn.linears.2.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.7.self_attn.linears.3.weight',\n",
              "              tensor([[-0.0180, -0.0294,  0.0508,  ..., -0.0282,  0.0035,  0.0425],\n",
              "                      [-0.0216,  0.0147,  0.0273,  ..., -0.0313, -0.0276,  0.0530],\n",
              "                      [-0.0540,  0.0457, -0.0288,  ...,  0.0480,  0.0489, -0.0200],\n",
              "                      ...,\n",
              "                      [-0.0180,  0.0042,  0.0379,  ..., -0.0349, -0.0202, -0.0470],\n",
              "                      [-0.0246,  0.0044, -0.0104,  ..., -0.0169,  0.0427,  0.0216],\n",
              "                      [-0.0389, -0.0376, -0.0253,  ...,  0.0520, -0.0338,  0.0322]])),\n",
              "             ('encoder.layers.7.self_attn.linears.3.bias',\n",
              "              tensor([ 0.0050,  0.0116, -0.0131,  ..., -0.0091,  0.0249,  0.0168])),\n",
              "             ('encoder.layers.7.feed_forward.linears.0.weight',\n",
              "              tensor([[ 0.0188,  0.0161,  0.0380,  ...,  0.0151, -0.0301,  0.0227],\n",
              "                      [-0.0428,  0.0472, -0.0003,  ...,  0.0389,  0.0242,  0.0414],\n",
              "                      [-0.0165, -0.0262, -0.0017,  ..., -0.0417, -0.0523,  0.0028],\n",
              "                      ...,\n",
              "                      [ 0.0479,  0.0298,  0.0072,  ...,  0.0162,  0.0154,  0.0340],\n",
              "                      [-0.0003, -0.0261, -0.0212,  ...,  0.0360, -0.0087,  0.0085],\n",
              "                      [ 0.0237, -0.0231, -0.0299,  ..., -0.0288, -0.0122, -0.0154]])),\n",
              "             ('encoder.layers.7.feed_forward.linears.0.bias',\n",
              "              tensor([-0.0011,  0.0296,  0.0139,  ..., -0.0297,  0.0307,  0.0093])),\n",
              "             ('encoder.layers.7.sublayer.0.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.7.sublayer.0.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.layers.7.sublayer.1.norm.a_2',\n",
              "              tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.layers.7.sublayer.1.norm.b_2',\n",
              "              tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('encoder.norm.a_2', tensor([1., 1., 1.,  ..., 1., 1., 1.])),\n",
              "             ('encoder.norm.b_2', tensor([0., 0., 0.,  ..., 0., 0., 0.])),\n",
              "             ('src_embed.lut.weight',\n",
              "              tensor([[-0.0360, -0.0648, -0.0262,  ..., -0.0299,  0.0411,  0.0619],\n",
              "                      [ 0.0472,  0.0045,  0.0036,  ...,  0.0361,  0.0173, -0.0443],\n",
              "                      [-0.0467,  0.0327,  0.0612,  ...,  0.0246,  0.0102,  0.0001],\n",
              "                      ...,\n",
              "                      [-0.0652,  0.0006,  0.0611,  ...,  0.0623,  0.0039,  0.0059],\n",
              "                      [ 0.0697, -0.0599,  0.0250,  ..., -0.0163, -0.0717, -0.0502],\n",
              "                      [-0.0008, -0.0014, -0.0332,  ...,  0.0471,  0.0346, -0.0067]])),\n",
              "             ('src_embed.lut.bias',\n",
              "              tensor([ 0.1070, -0.0827, -0.1215,  ..., -0.0212, -0.0274, -0.1354])),\n",
              "             ('generator.proj.weight',\n",
              "              tensor([[ 0.0559, -0.0465, -0.0058,  ...,  0.0383,  0.0343, -0.0408]])),\n",
              "             ('generator.proj.bias', tensor([0.0085]))])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "js1UnrRg3MNE",
        "outputId": "8361cf86-4314-42b2-8dc5-6e9c5f089199",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model_state_dict"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "OrderedDict([('encoder.layers.0.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0523,  0.0197, -0.0003,  ...,  0.0171,  0.0427, -0.0305],\n",
              "                      [ 0.0014,  0.0492, -0.0425,  ...,  0.0028, -0.0104,  0.0015],\n",
              "                      [ 0.0403,  0.0453,  0.0339,  ...,  0.0180, -0.0286, -0.0413],\n",
              "                      ...,\n",
              "                      [ 0.0378,  0.0147,  0.0352,  ...,  0.0411, -0.0403,  0.0409],\n",
              "                      [ 0.0477,  0.0430,  0.0343,  ...,  0.0194,  0.0390, -0.0340],\n",
              "                      [-0.0023, -0.0492, -0.0140,  ..., -0.0462,  0.0443, -0.0047]])),\n",
              "             ('encoder.layers.0.self_attn.linears.0.bias',\n",
              "              tensor([-0.0189,  0.0018,  0.0261,  ...,  0.0225, -0.0111, -0.0002])),\n",
              "             ('encoder.layers.0.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0003, -0.0490, -0.0487,  ...,  0.0246,  0.0356, -0.0149],\n",
              "                      [ 0.0270,  0.0260, -0.0352,  ...,  0.0345, -0.0443, -0.0432],\n",
              "                      [-0.0615,  0.0374, -0.0264,  ...,  0.0177,  0.0370, -0.0379],\n",
              "                      ...,\n",
              "                      [-0.0119, -0.0468, -0.0118,  ...,  0.0114,  0.0360, -0.0005],\n",
              "                      [ 0.0275,  0.0526, -0.0453,  ..., -0.0517, -0.0019, -0.0282],\n",
              "                      [-0.0230, -0.0299,  0.0233,  ..., -0.0159, -0.0045, -0.0058]])),\n",
              "             ('encoder.layers.0.self_attn.linears.1.bias',\n",
              "              tensor([-0.0168,  0.0088,  0.0264,  ...,  0.0202, -0.0113,  0.0006])),\n",
              "             ('encoder.layers.0.self_attn.linears.2.weight',\n",
              "              tensor([[ 0.0513, -0.0106, -0.0412,  ...,  0.0180, -0.0074,  0.0028],\n",
              "                      [-0.0009, -0.0138, -0.0422,  ..., -0.0180, -0.0037, -0.0056],\n",
              "                      [-0.0270,  0.0103, -0.0265,  ...,  0.0260,  0.0143,  0.0044],\n",
              "                      ...,\n",
              "                      [ 0.0060,  0.0267,  0.0219,  ...,  0.0316, -0.0099,  0.0610],\n",
              "                      [ 0.0131,  0.0128,  0.0246,  ..., -0.0120, -0.0461, -0.0189],\n",
              "                      [-0.0096, -0.0381,  0.0436,  ..., -0.0226,  0.0010, -0.0450]])),\n",
              "             ('encoder.layers.0.self_attn.linears.2.bias',\n",
              "              tensor([-0.0160,  0.0083,  0.0266,  ...,  0.0216, -0.0120,  0.0019])),\n",
              "             ('encoder.layers.0.self_attn.linears.3.weight',\n",
              "              tensor([[-0.0560, -0.0033,  0.0278,  ..., -0.0211,  0.0405,  0.0467],\n",
              "                      [-0.0049,  0.0341, -0.0418,  ...,  0.0566, -0.0022, -0.0452],\n",
              "                      [ 0.0313,  0.0056,  0.0162,  ...,  0.0435, -0.0417,  0.0092],\n",
              "                      ...,\n",
              "                      [-0.0346, -0.0077, -0.0521,  ..., -0.0144,  0.0439, -0.0430],\n",
              "                      [-0.0262, -0.0225, -0.0285,  ..., -0.0320,  0.0212,  0.0063],\n",
              "                      [ 0.0445,  0.0331, -0.0143,  ...,  0.0089,  0.0076, -0.0104]])),\n",
              "             ('encoder.layers.0.self_attn.linears.3.bias',\n",
              "              tensor([-0.0188,  0.0086,  0.0284,  ...,  0.0211, -0.0092, -0.0046])),\n",
              "             ('encoder.layers.0.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0506, -0.0421, -0.0479,  ..., -0.0197, -0.0440, -0.0083],\n",
              "                      [-0.0126, -0.0443, -0.0563,  ..., -0.0034,  0.0641,  0.0497],\n",
              "                      [-0.0234, -0.0294, -0.0069,  ..., -0.0423, -0.0396, -0.0260],\n",
              "                      ...,\n",
              "                      [-0.0162, -0.0506, -0.0119,  ...,  0.0080,  0.0319, -0.0441],\n",
              "                      [-0.0266,  0.0057, -0.0084,  ..., -0.0529, -0.0490,  0.0214],\n",
              "                      [-0.0172, -0.0597, -0.0315,  ..., -0.0059, -0.0340,  0.0187]])),\n",
              "             ('encoder.layers.0.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0004, -0.0240,  0.0229,  ...,  0.0229, -0.0180, -0.0021])),\n",
              "             ('encoder.layers.0.sublayer.0.norm.a_2',\n",
              "              tensor([0.9977, 0.9980, 1.0044,  ..., 0.9923, 0.9925, 1.0037])),\n",
              "             ('encoder.layers.0.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0014, -0.0046, -0.0007,  ..., -0.0086, -0.0035,  0.0059])),\n",
              "             ('encoder.layers.0.sublayer.1.norm.a_2',\n",
              "              tensor([1.0061, 1.0072, 0.9989,  ..., 1.0021, 1.0076, 1.0108])),\n",
              "             ('encoder.layers.0.sublayer.1.norm.b_2',\n",
              "              tensor([-3.5557e-04,  2.7526e-03,  2.9475e-03,  ..., -3.4370e-03,\n",
              "                       9.6726e-05, -6.6116e-03])),\n",
              "             ('encoder.layers.1.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0058,  0.0317, -0.0087,  ...,  0.0059, -0.0506, -0.0272],\n",
              "                      [ 0.0048, -0.0072, -0.0316,  ...,  0.0263,  0.0309, -0.0445],\n",
              "                      [-0.0364, -0.0046,  0.0322,  ..., -0.0404,  0.0353, -0.0544],\n",
              "                      ...,\n",
              "                      [ 0.0172,  0.0094, -0.0479,  ..., -0.0341, -0.0320, -0.0480],\n",
              "                      [-0.0272, -0.0082, -0.0083,  ...,  0.0484,  0.0451, -0.0088],\n",
              "                      [ 0.0296, -0.0205, -0.0135,  ...,  0.0535, -0.0251,  0.0442]])),\n",
              "             ('encoder.layers.1.self_attn.linears.0.bias',\n",
              "              tensor([-0.0224,  0.0210,  0.0330,  ...,  0.0189, -0.0095, -0.0153])),\n",
              "             ('encoder.layers.1.self_attn.linears.1.weight',\n",
              "              tensor([[ 0.0413,  0.0533, -0.0137,  ...,  0.0170,  0.0414,  0.0217],\n",
              "                      [ 0.0226,  0.0157,  0.0269,  ..., -0.0220,  0.0043,  0.0236],\n",
              "                      [ 0.0400, -0.0283,  0.0573,  ..., -0.0514, -0.0143, -0.0349],\n",
              "                      ...,\n",
              "                      [ 0.0145, -0.0113, -0.0299,  ..., -0.0214,  0.0281, -0.0521],\n",
              "                      [ 0.0049,  0.0177, -0.0181,  ..., -0.0077,  0.0348, -0.0238],\n",
              "                      [-0.0076, -0.0046,  0.0186,  ..., -0.0428, -0.0180,  0.0013]])),\n",
              "             ('encoder.layers.1.self_attn.linears.1.bias',\n",
              "              tensor([-1.6915e-02,  8.6603e-03,  2.6303e-02,  ...,  2.1149e-02,\n",
              "                      -1.1799e-02, -9.4858e-05])),\n",
              "             ('encoder.layers.1.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0081, -0.0458, -0.0108,  ...,  0.0074,  0.0480,  0.0375],\n",
              "                      [ 0.0047, -0.0504, -0.0232,  ..., -0.0167, -0.0065, -0.0195],\n",
              "                      [-0.0011, -0.0429, -0.0443,  ..., -0.0282, -0.0341, -0.0483],\n",
              "                      ...,\n",
              "                      [ 0.0134, -0.0006,  0.0407,  ..., -0.0464,  0.0212, -0.0308],\n",
              "                      [-0.0286,  0.0223,  0.0097,  ...,  0.0165,  0.0002,  0.0111],\n",
              "                      [ 0.0373,  0.0267, -0.0069,  ..., -0.0259, -0.0409, -0.0451]])),\n",
              "             ('encoder.layers.1.self_attn.linears.2.bias',\n",
              "              tensor([-0.0123,  0.0073,  0.0237,  ...,  0.0215, -0.0097,  0.0022])),\n",
              "             ('encoder.layers.1.self_attn.linears.3.weight',\n",
              "              tensor([[-4.6729e-02, -4.0582e-02, -1.2572e-02,  ..., -7.3779e-03,\n",
              "                       -3.6672e-03, -4.6826e-02],\n",
              "                      [ 4.7870e-02, -3.9885e-02, -5.2955e-03,  ...,  1.6887e-02,\n",
              "                       -6.5864e-03, -1.0508e-02],\n",
              "                      [-2.1034e-02,  4.6441e-02,  4.1189e-02,  ..., -2.4511e-02,\n",
              "                       -1.7949e-02,  1.6522e-02],\n",
              "                      ...,\n",
              "                      [ 2.7128e-02,  3.3623e-02, -2.9111e-03,  ...,  4.1705e-02,\n",
              "                        4.5360e-02, -4.5868e-02],\n",
              "                      [ 5.2379e-02,  2.5552e-02,  2.7708e-02,  ..., -1.8306e-02,\n",
              "                       -6.3889e-06, -2.0187e-02],\n",
              "                      [-4.0883e-02,  3.6099e-02, -4.6619e-02,  ...,  3.2978e-02,\n",
              "                       -3.8937e-02, -9.4912e-04]])),\n",
              "             ('encoder.layers.1.self_attn.linears.3.bias',\n",
              "              tensor([-0.0164,  0.0076,  0.0263,  ...,  0.0231, -0.0089, -0.0002])),\n",
              "             ('encoder.layers.1.feed_forward.linears.0.weight',\n",
              "              tensor([[ 0.0083, -0.0142, -0.0514,  ..., -0.0187, -0.0126, -0.0228],\n",
              "                      [-0.0080, -0.0144,  0.0443,  ..., -0.0397,  0.0376,  0.0120],\n",
              "                      [-0.0422, -0.0519,  0.0174,  ...,  0.0249,  0.0363,  0.0388],\n",
              "                      ...,\n",
              "                      [-0.0145, -0.0225,  0.0291,  ..., -0.0278,  0.0106,  0.0086],\n",
              "                      [ 0.0054, -0.0045,  0.0353,  ..., -0.0562, -0.0349, -0.0447],\n",
              "                      [-0.0239, -0.0428,  0.0378,  ...,  0.0260, -0.0237,  0.0414]])),\n",
              "             ('encoder.layers.1.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0046, -0.0287,  0.0204,  ...,  0.0235, -0.0198, -0.0013])),\n",
              "             ('encoder.layers.1.sublayer.0.norm.a_2',\n",
              "              tensor([0.9919, 0.9947, 1.0000,  ..., 0.9882, 0.9950, 1.0061])),\n",
              "             ('encoder.layers.1.sublayer.0.norm.b_2',\n",
              "              tensor([-0.0050, -0.0032, -0.0024,  ...,  0.0022,  0.0028, -0.0025])),\n",
              "             ('encoder.layers.1.sublayer.1.norm.a_2',\n",
              "              tensor([0.9985, 1.0121, 1.0007,  ..., 0.9999, 1.0065, 1.0168])),\n",
              "             ('encoder.layers.1.sublayer.1.norm.b_2',\n",
              "              tensor([-2.8897e-03, -9.1054e-05,  1.4950e-03,  ..., -1.4927e-03,\n",
              "                       4.7737e-03,  1.8817e-03])),\n",
              "             ('encoder.layers.2.self_attn.linears.0.weight',\n",
              "              tensor([[-0.0101, -0.0493,  0.0050,  ..., -0.0254, -0.0141,  0.0204],\n",
              "                      [-0.0189, -0.0531,  0.0157,  ...,  0.0378,  0.0156,  0.0625],\n",
              "                      [-0.0169, -0.0268, -0.0404,  ..., -0.0152,  0.0473, -0.0178],\n",
              "                      ...,\n",
              "                      [-0.0400, -0.0446,  0.0154,  ..., -0.0317,  0.0412,  0.0367],\n",
              "                      [ 0.0076,  0.0159,  0.0042,  ..., -0.0373,  0.0235,  0.0490],\n",
              "                      [-0.0350,  0.0447, -0.0222,  ..., -0.0184,  0.0293,  0.0004]])),\n",
              "             ('encoder.layers.2.self_attn.linears.0.bias',\n",
              "              tensor([-0.0314, -0.0093,  0.0136,  ...,  0.0193, -0.0179, -0.0004])),\n",
              "             ('encoder.layers.2.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0079,  0.0436,  0.0320,  ...,  0.0028,  0.0585, -0.0308],\n",
              "                      [ 0.0266,  0.0031, -0.0612,  ..., -0.0623, -0.0326, -0.0166],\n",
              "                      [-0.0008,  0.0163, -0.0268,  ...,  0.0260,  0.0578, -0.0117],\n",
              "                      ...,\n",
              "                      [-0.0166, -0.0274, -0.0531,  ...,  0.0384,  0.0221, -0.0404],\n",
              "                      [-0.0414, -0.0340,  0.0337,  ...,  0.0532,  0.0399, -0.0166],\n",
              "                      [-0.0315,  0.0051,  0.0113,  ..., -0.0418, -0.0366,  0.0497]])),\n",
              "             ('encoder.layers.2.self_attn.linears.1.bias',\n",
              "              tensor([-0.0169,  0.0087,  0.0265,  ...,  0.0211, -0.0118, -0.0002])),\n",
              "             ('encoder.layers.2.self_attn.linears.2.weight',\n",
              "              tensor([[ 0.0368,  0.0275,  0.0282,  ...,  0.0029,  0.0603,  0.0355],\n",
              "                      [-0.0415,  0.0477, -0.0363,  ..., -0.0465,  0.0457, -0.0163],\n",
              "                      [-0.0243, -0.0168,  0.0235,  ...,  0.0148, -0.0163, -0.0112],\n",
              "                      ...,\n",
              "                      [-0.0502,  0.0435, -0.0503,  ...,  0.0321, -0.0273, -0.0458],\n",
              "                      [ 0.0158,  0.0221,  0.0204,  ...,  0.0038,  0.0186, -0.0210],\n",
              "                      [-0.0380, -0.0383, -0.0075,  ...,  0.0260,  0.0184, -0.0050]])),\n",
              "             ('encoder.layers.2.self_attn.linears.2.bias',\n",
              "              tensor([-0.0138,  0.0119,  0.0270,  ...,  0.0208, -0.0120, -0.0037])),\n",
              "             ('encoder.layers.2.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0048, -0.0397,  0.0189,  ...,  0.0018,  0.0195,  0.0059],\n",
              "                      [ 0.0332, -0.0151, -0.0172,  ...,  0.0238,  0.0137,  0.0065],\n",
              "                      [ 0.0109, -0.0193, -0.0578,  ...,  0.0315, -0.0372,  0.0433],\n",
              "                      ...,\n",
              "                      [-0.0547,  0.0438,  0.0118,  ..., -0.0535,  0.0132, -0.0280],\n",
              "                      [-0.0134, -0.0481,  0.0198,  ...,  0.0139, -0.0144, -0.0419],\n",
              "                      [ 0.0044,  0.0357, -0.0154,  ..., -0.0409, -0.0140,  0.0077]])),\n",
              "             ('encoder.layers.2.self_attn.linears.3.bias',\n",
              "              tensor([-0.0171,  0.0069,  0.0249,  ...,  0.0264, -0.0093, -0.0006])),\n",
              "             ('encoder.layers.2.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0218, -0.0095,  0.0272,  ..., -0.0309,  0.0162, -0.0091],\n",
              "                      [ 0.0455, -0.0561, -0.0350,  ...,  0.0010,  0.0445, -0.0212],\n",
              "                      [ 0.0059,  0.0497,  0.0022,  ...,  0.0441,  0.0211,  0.0321],\n",
              "                      ...,\n",
              "                      [ 0.0254,  0.0460, -0.0509,  ..., -0.0237,  0.0442,  0.0248],\n",
              "                      [-0.0275,  0.0378,  0.0099,  ..., -0.0520, -0.0496, -0.0066],\n",
              "                      [-0.0134, -0.0217, -0.0475,  ..., -0.0116, -0.0012, -0.0275]])),\n",
              "             ('encoder.layers.2.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0002, -0.0322,  0.0231,  ...,  0.0239, -0.0200, -0.0045])),\n",
              "             ('encoder.layers.2.sublayer.0.norm.a_2',\n",
              "              tensor([1.0012, 0.9998, 1.0066,  ..., 1.0013, 1.0034, 0.9976])),\n",
              "             ('encoder.layers.2.sublayer.0.norm.b_2',\n",
              "              tensor([-2.0655e-04, -1.3716e-03, -2.1849e-03,  ..., -2.7905e-03,\n",
              "                       1.6901e-03, -4.1606e-05])),\n",
              "             ('encoder.layers.2.sublayer.1.norm.a_2',\n",
              "              tensor([1.0017, 1.0030, 1.0042,  ..., 1.0084, 1.0054, 1.0083])),\n",
              "             ('encoder.layers.2.sublayer.1.norm.b_2',\n",
              "              tensor([ 3.0622e-03, -3.7244e-03, -3.9762e-03,  ...,  3.5263e-05,\n",
              "                       1.2276e-03,  8.5568e-04])),\n",
              "             ('encoder.layers.3.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0319,  0.0499, -0.0031,  ...,  0.0044, -0.0461,  0.0397],\n",
              "                      [ 0.0397,  0.0355,  0.0414,  ..., -0.0445,  0.0021, -0.0440],\n",
              "                      [-0.0583, -0.0248,  0.0236,  ..., -0.0073, -0.0348,  0.0017],\n",
              "                      ...,\n",
              "                      [ 0.0326, -0.0527, -0.0175,  ..., -0.0128,  0.0046, -0.0341],\n",
              "                      [-0.0129, -0.0490, -0.0429,  ...,  0.0054,  0.0394,  0.0250],\n",
              "                      [-0.0364,  0.0059, -0.0284,  ...,  0.0323, -0.0212,  0.0112]])),\n",
              "             ('encoder.layers.3.self_attn.linears.0.bias',\n",
              "              tensor([-0.0145,  0.0141,  0.0298,  ...,  0.0135, -0.0126, -0.0152])),\n",
              "             ('encoder.layers.3.self_attn.linears.1.weight',\n",
              "              tensor([[ 0.0215, -0.0113, -0.0088,  ..., -0.0442,  0.0532,  0.0529],\n",
              "                      [-0.0259, -0.0006, -0.0051,  ...,  0.0455,  0.0390, -0.0011],\n",
              "                      [ 0.0380, -0.0447,  0.0308,  ...,  0.0332, -0.0350,  0.0312],\n",
              "                      ...,\n",
              "                      [-0.0158,  0.0258, -0.0260,  ..., -0.0259,  0.0269, -0.0299],\n",
              "                      [ 0.0416, -0.0022,  0.0146,  ..., -0.0509,  0.0307, -0.0451],\n",
              "                      [ 0.0449, -0.0019,  0.0014,  ..., -0.0621, -0.0162,  0.0481]])),\n",
              "             ('encoder.layers.3.self_attn.linears.1.bias',\n",
              "              tensor([-0.0169,  0.0087,  0.0264,  ...,  0.0210, -0.0118, -0.0002])),\n",
              "             ('encoder.layers.3.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0069, -0.0098,  0.0224,  ..., -0.0018, -0.0512,  0.0456],\n",
              "                      [ 0.0167, -0.0118, -0.0542,  ...,  0.0216,  0.0075,  0.0303],\n",
              "                      [-0.0127,  0.0332, -0.0273,  ..., -0.0328,  0.0388, -0.0019],\n",
              "                      ...,\n",
              "                      [ 0.0016, -0.0227,  0.0455,  ..., -0.0240, -0.0488, -0.0231],\n",
              "                      [ 0.0467, -0.0070, -0.0386,  ...,  0.0251, -0.0013, -0.0489],\n",
              "                      [ 0.0109, -0.0098, -0.0062,  ..., -0.0381, -0.0083,  0.0272]])),\n",
              "             ('encoder.layers.3.self_attn.linears.2.bias',\n",
              "              tensor([-0.0204,  0.0079,  0.0243,  ...,  0.0234, -0.0111,  0.0021])),\n",
              "             ('encoder.layers.3.self_attn.linears.3.weight',\n",
              "              tensor([[-0.0039,  0.0293, -0.0227,  ...,  0.0502,  0.0071, -0.0070],\n",
              "                      [-0.0297, -0.0242,  0.0135,  ...,  0.0502,  0.0495,  0.0204],\n",
              "                      [-0.0581,  0.0398, -0.0397,  ..., -0.0377,  0.0286, -0.0321],\n",
              "                      ...,\n",
              "                      [ 0.0065,  0.0536,  0.0358,  ...,  0.0421, -0.0244, -0.0213],\n",
              "                      [ 0.0413,  0.0450, -0.0432,  ..., -0.0189, -0.0399,  0.0052],\n",
              "                      [ 0.0566,  0.0295, -0.0447,  ..., -0.0404, -0.0407, -0.0051]])),\n",
              "             ('encoder.layers.3.self_attn.linears.3.bias',\n",
              "              tensor([-0.0168,  0.0057,  0.0244,  ...,  0.0241, -0.0118, -0.0002])),\n",
              "             ('encoder.layers.3.feed_forward.linears.0.weight',\n",
              "              tensor([[ 0.0058,  0.0258, -0.0403,  ..., -0.0274,  0.0353, -0.0139],\n",
              "                      [ 0.0098,  0.0245,  0.0188,  ..., -0.0299,  0.0292,  0.0160],\n",
              "                      [ 0.0409, -0.0099,  0.0306,  ..., -0.0511, -0.0462, -0.0615],\n",
              "                      ...,\n",
              "                      [ 0.0084, -0.0595, -0.0301,  ...,  0.0046, -0.0028,  0.0490],\n",
              "                      [ 0.0321,  0.0437, -0.0159,  ...,  0.0246, -0.0632,  0.0463],\n",
              "                      [-0.0024,  0.0072, -0.0487,  ...,  0.0658, -0.0234, -0.0159]])),\n",
              "             ('encoder.layers.3.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0029, -0.0293,  0.0250,  ...,  0.0202, -0.0225,  0.0026])),\n",
              "             ('encoder.layers.3.sublayer.0.norm.a_2',\n",
              "              tensor([0.9957, 0.9973, 1.0031,  ..., 1.0008, 1.0010, 1.0012])),\n",
              "             ('encoder.layers.3.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0031, -0.0004, -0.0051,  ..., -0.0002,  0.0020, -0.0013])),\n",
              "             ('encoder.layers.3.sublayer.1.norm.a_2',\n",
              "              tensor([1.0045, 1.0125, 0.9995,  ..., 1.0043, 1.0264, 1.0087])),\n",
              "             ('encoder.layers.3.sublayer.1.norm.b_2',\n",
              "              tensor([-0.0023,  0.0047, -0.0056,  ...,  0.0019,  0.0032, -0.0043])),\n",
              "             ('encoder.layers.4.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0253,  0.0171, -0.0514,  ..., -0.0209,  0.0183,  0.0210],\n",
              "                      [ 0.0171, -0.0219,  0.0169,  ...,  0.0272, -0.0446, -0.0577],\n",
              "                      [-0.0318,  0.0403,  0.0295,  ...,  0.0184,  0.0276, -0.0614],\n",
              "                      ...,\n",
              "                      [ 0.0439,  0.0350, -0.0427,  ...,  0.0010,  0.0060,  0.0200],\n",
              "                      [ 0.0029,  0.0153,  0.0342,  ...,  0.0242,  0.0393, -0.0020],\n",
              "                      [-0.0227,  0.0438,  0.0516,  ..., -0.0276, -0.0528,  0.0308]])),\n",
              "             ('encoder.layers.4.self_attn.linears.0.bias',\n",
              "              tensor([-0.0156,  0.0064,  0.0204,  ...,  0.0234, -0.0103,  0.0045])),\n",
              "             ('encoder.layers.4.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0288,  0.0543,  0.0070,  ...,  0.0349,  0.0072, -0.0102],\n",
              "                      [-0.0500,  0.0073, -0.0156,  ...,  0.0365, -0.0142,  0.0250],\n",
              "                      [-0.0012,  0.0533,  0.0042,  ..., -0.0422, -0.0356,  0.0050],\n",
              "                      ...,\n",
              "                      [ 0.0197,  0.0074,  0.0445,  ..., -0.0508, -0.0342, -0.0035],\n",
              "                      [-0.0494,  0.0441,  0.0169,  ...,  0.0384,  0.0268, -0.0085],\n",
              "                      [ 0.0126,  0.0400, -0.0461,  ...,  0.0330, -0.0261,  0.0070]])),\n",
              "             ('encoder.layers.4.self_attn.linears.1.bias',\n",
              "              tensor([-1.7398e-02,  9.2496e-03,  2.6399e-02,  ...,  2.1093e-02,\n",
              "                      -1.1747e-02, -7.2788e-05])),\n",
              "             ('encoder.layers.4.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0083,  0.0324, -0.0212,  ..., -0.0515,  0.0332, -0.0226],\n",
              "                      [-0.0149,  0.0210, -0.0424,  ...,  0.0415, -0.0326, -0.0116],\n",
              "                      [ 0.0047, -0.0218, -0.0141,  ..., -0.0086, -0.0041, -0.0294],\n",
              "                      ...,\n",
              "                      [-0.0279,  0.0224, -0.0048,  ..., -0.0038,  0.0201,  0.0550],\n",
              "                      [-0.0148,  0.0468,  0.0362,  ..., -0.0434, -0.0404, -0.0323],\n",
              "                      [ 0.0308, -0.0308, -0.0412,  ...,  0.0235, -0.0410,  0.0057]])),\n",
              "             ('encoder.layers.4.self_attn.linears.2.bias',\n",
              "              tensor([-0.0190,  0.0078,  0.0272,  ...,  0.0240, -0.0166, -0.0009])),\n",
              "             ('encoder.layers.4.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0367, -0.0441, -0.0380,  ...,  0.0201, -0.0433, -0.0020],\n",
              "                      [-0.0007,  0.0158, -0.0347,  ...,  0.0006,  0.0452,  0.0109],\n",
              "                      [-0.0512, -0.0328,  0.0350,  ..., -0.0377, -0.0366, -0.0514],\n",
              "                      ...,\n",
              "                      [-0.0309,  0.0495,  0.0260,  ..., -0.0406, -0.0505, -0.0103],\n",
              "                      [ 0.0358,  0.0385, -0.0174,  ..., -0.0459, -0.0351,  0.0157],\n",
              "                      [-0.0244,  0.0129,  0.0485,  ..., -0.0480, -0.0103,  0.0401]])),\n",
              "             ('encoder.layers.4.self_attn.linears.3.bias',\n",
              "              tensor([-0.0153,  0.0077,  0.0234,  ...,  0.0225, -0.0141,  0.0002])),\n",
              "             ('encoder.layers.4.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0209,  0.0098, -0.0379,  ...,  0.0503,  0.0028,  0.0306],\n",
              "                      [ 0.0213, -0.0210,  0.0236,  ...,  0.0233, -0.0043,  0.0170],\n",
              "                      [ 0.0322,  0.0399, -0.0390,  ...,  0.0025,  0.0069,  0.0238],\n",
              "                      ...,\n",
              "                      [ 0.0043, -0.0115,  0.0202,  ..., -0.0584, -0.0021, -0.0234],\n",
              "                      [-0.0103,  0.0375, -0.0282,  ...,  0.0285, -0.0110,  0.0079],\n",
              "                      [ 0.0409,  0.0146,  0.0218,  ..., -0.0338, -0.0311, -0.0454]])),\n",
              "             ('encoder.layers.4.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0076, -0.0279,  0.0205,  ...,  0.0182, -0.0240, -0.0023])),\n",
              "             ('encoder.layers.4.sublayer.0.norm.a_2',\n",
              "              tensor([0.9926, 0.9982, 1.0083,  ..., 1.0012, 0.9946, 1.0045])),\n",
              "             ('encoder.layers.4.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0003, -0.0025, -0.0015,  ...,  0.0017,  0.0001,  0.0034])),\n",
              "             ('encoder.layers.4.sublayer.1.norm.a_2',\n",
              "              tensor([1.0020, 1.0145, 0.9950,  ..., 1.0035, 1.0081, 1.0067])),\n",
              "             ('encoder.layers.4.sublayer.1.norm.b_2',\n",
              "              tensor([-0.0002, -0.0046, -0.0042,  ..., -0.0003, -0.0003,  0.0016])),\n",
              "             ('encoder.layers.5.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0027,  0.0502, -0.0187,  ...,  0.0299,  0.0314,  0.0078],\n",
              "                      [-0.0251,  0.0437, -0.0397,  ...,  0.0506, -0.0228, -0.0229],\n",
              "                      [ 0.0154,  0.0432,  0.0349,  ..., -0.0066, -0.0396, -0.0124],\n",
              "                      ...,\n",
              "                      [ 0.0091, -0.0253, -0.0474,  ...,  0.0588, -0.0450, -0.0334],\n",
              "                      [-0.0556, -0.0421, -0.0020,  ..., -0.0450,  0.0145, -0.0437],\n",
              "                      [-0.0217,  0.0089,  0.0304,  ...,  0.0197, -0.0511,  0.0352]])),\n",
              "             ('encoder.layers.5.self_attn.linears.0.bias',\n",
              "              tensor([-0.0098,  0.0174,  0.0225,  ...,  0.0143, -0.0189,  0.0054])),\n",
              "             ('encoder.layers.5.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0015, -0.0358, -0.0015,  ..., -0.0038,  0.0441, -0.0327],\n",
              "                      [ 0.0137, -0.0028, -0.0169,  ...,  0.0204,  0.0142,  0.0098],\n",
              "                      [ 0.0032, -0.0543,  0.0425,  ..., -0.0071,  0.0007,  0.0376],\n",
              "                      ...,\n",
              "                      [-0.0436,  0.0523,  0.0040,  ..., -0.0228, -0.0334, -0.0586],\n",
              "                      [ 0.0170,  0.0060, -0.0216,  ...,  0.0443,  0.0056,  0.0371],\n",
              "                      [ 0.0223,  0.0323,  0.0508,  ...,  0.0136,  0.0379,  0.0044]])),\n",
              "             ('encoder.layers.5.self_attn.linears.1.bias',\n",
              "              tensor([-0.0168,  0.0088,  0.0264,  ...,  0.0211, -0.0118, -0.0002])),\n",
              "             ('encoder.layers.5.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0425,  0.0460,  0.0385,  ..., -0.0148, -0.0283,  0.0217],\n",
              "                      [ 0.0533,  0.0393,  0.0475,  ..., -0.0485, -0.0464, -0.0103],\n",
              "                      [-0.0070,  0.0019, -0.0132,  ...,  0.0159,  0.0513,  0.0451],\n",
              "                      ...,\n",
              "                      [ 0.0388,  0.0434,  0.0128,  ...,  0.0197,  0.0207, -0.0462],\n",
              "                      [-0.0022, -0.0250,  0.0446,  ..., -0.0478,  0.0076,  0.0253],\n",
              "                      [-0.0069, -0.0084,  0.0059,  ...,  0.0200, -0.0180,  0.0260]])),\n",
              "             ('encoder.layers.5.self_attn.linears.2.bias',\n",
              "              tensor([-0.0152,  0.0127,  0.0281,  ...,  0.0209, -0.0128, -0.0005])),\n",
              "             ('encoder.layers.5.self_attn.linears.3.weight',\n",
              "              tensor([[-0.0060,  0.0208, -0.0342,  ..., -0.0338,  0.0132,  0.0479],\n",
              "                      [-0.0251, -0.0093,  0.0258,  ..., -0.0174,  0.0243,  0.0207],\n",
              "                      [ 0.0047,  0.0304,  0.0428,  ..., -0.0205,  0.0261,  0.0494],\n",
              "                      ...,\n",
              "                      [ 0.0035,  0.0259, -0.0148,  ..., -0.0066,  0.0107, -0.0110],\n",
              "                      [-0.0462,  0.0050,  0.0192,  ..., -0.0212, -0.0311, -0.0309],\n",
              "                      [ 0.0459, -0.0570,  0.0215,  ...,  0.0197,  0.0279,  0.0494]])),\n",
              "             ('encoder.layers.5.self_attn.linears.3.bias',\n",
              "              tensor([-0.0134,  0.0064,  0.0231,  ...,  0.0222, -0.0104,  0.0015])),\n",
              "             ('encoder.layers.5.feed_forward.linears.0.weight',\n",
              "              tensor([[ 0.0322, -0.0207,  0.0014,  ..., -0.0066, -0.0540, -0.0087],\n",
              "                      [-0.0287, -0.0260, -0.0004,  ..., -0.0117, -0.0348, -0.0507],\n",
              "                      [ 0.0149, -0.0258, -0.0524,  ...,  0.0166, -0.0213,  0.0055],\n",
              "                      ...,\n",
              "                      [-0.0036,  0.0255, -0.0065,  ...,  0.0190, -0.0334,  0.0291],\n",
              "                      [ 0.0301,  0.0567,  0.0051,  ...,  0.0027,  0.0168, -0.0396],\n",
              "                      [ 0.0073, -0.0377,  0.0456,  ...,  0.0027, -0.0189,  0.0249]])),\n",
              "             ('encoder.layers.5.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0015, -0.0295,  0.0203,  ...,  0.0217, -0.0208, -0.0018])),\n",
              "             ('encoder.layers.5.sublayer.0.norm.a_2',\n",
              "              tensor([0.9922, 0.9949, 1.0026,  ..., 0.9877, 0.9931, 1.0017])),\n",
              "             ('encoder.layers.5.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0007, -0.0020, -0.0016,  ...,  0.0021, -0.0066,  0.0013])),\n",
              "             ('encoder.layers.5.sublayer.1.norm.a_2',\n",
              "              tensor([1.0059, 1.0031, 1.0133,  ..., 1.0238, 1.0100, 1.0167])),\n",
              "             ('encoder.layers.5.sublayer.1.norm.b_2',\n",
              "              tensor([ 0.0040, -0.0036,  0.0013,  ..., -0.0009,  0.0002,  0.0028])),\n",
              "             ('encoder.layers.6.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0304,  0.0387, -0.0403,  ..., -0.0464,  0.0049, -0.0196],\n",
              "                      [ 0.0325, -0.0084,  0.0333,  ..., -0.0223, -0.0414,  0.0308],\n",
              "                      [ 0.0386, -0.0249,  0.0061,  ..., -0.0505, -0.0548, -0.0374],\n",
              "                      ...,\n",
              "                      [-0.0344,  0.0492, -0.0227,  ..., -0.0550,  0.0393,  0.0233],\n",
              "                      [ 0.0158, -0.0272,  0.0272,  ...,  0.0148, -0.0299, -0.0482],\n",
              "                      [-0.0181, -0.0019,  0.0395,  ..., -0.0365,  0.0215,  0.0212]])),\n",
              "             ('encoder.layers.6.self_attn.linears.0.bias',\n",
              "              tensor([-0.0152,  0.0108,  0.0250,  ...,  0.0174, -0.0187, -0.0027])),\n",
              "             ('encoder.layers.6.self_attn.linears.1.weight',\n",
              "              tensor([[ 0.0475,  0.0228, -0.0018,  ...,  0.0282,  0.0225,  0.0517],\n",
              "                      [-0.0002, -0.0437,  0.0307,  ...,  0.0288, -0.0458,  0.0360],\n",
              "                      [ 0.0149,  0.0358,  0.0313,  ..., -0.0012, -0.0137, -0.0401],\n",
              "                      ...,\n",
              "                      [ 0.0468,  0.0029, -0.0480,  ...,  0.0063, -0.0168,  0.0569],\n",
              "                      [ 0.0486,  0.0068,  0.0048,  ...,  0.0004, -0.0283,  0.0581],\n",
              "                      [ 0.0106, -0.0050, -0.0381,  ...,  0.0137,  0.0040,  0.0550]])),\n",
              "             ('encoder.layers.6.self_attn.linears.1.bias',\n",
              "              tensor([-0.0169,  0.0087,  0.0264,  ...,  0.0211, -0.0118, -0.0001])),\n",
              "             ('encoder.layers.6.self_attn.linears.2.weight',\n",
              "              tensor([[ 2.2013e-02,  2.5594e-02, -1.8543e-02,  ...,  4.4537e-04,\n",
              "                        1.2771e-02,  4.7090e-02],\n",
              "                      [-1.5438e-02,  5.5459e-03, -5.1401e-02,  ..., -3.6312e-02,\n",
              "                        2.8141e-02, -2.0746e-02],\n",
              "                      [-1.2342e-03,  3.7859e-02,  4.0668e-02,  ..., -5.3282e-04,\n",
              "                       -1.2893e-02,  1.0674e-02],\n",
              "                      ...,\n",
              "                      [ 2.4686e-02, -4.0530e-02, -8.7064e-06,  ...,  4.7257e-02,\n",
              "                        1.5868e-02, -2.0252e-04],\n",
              "                      [ 4.3792e-02, -3.4945e-02, -7.3483e-03,  ..., -1.6980e-02,\n",
              "                       -2.6048e-02, -4.6858e-02],\n",
              "                      [ 3.6992e-02, -9.2648e-04, -3.4188e-02,  ..., -1.1160e-02,\n",
              "                        4.4632e-02,  4.1095e-02]])),\n",
              "             ('encoder.layers.6.self_attn.linears.2.bias',\n",
              "              tensor([-0.0172,  0.0116,  0.0260,  ...,  0.0206, -0.0094,  0.0012])),\n",
              "             ('encoder.layers.6.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0433,  0.0168,  0.0196,  ...,  0.0170,  0.0531,  0.0564],\n",
              "                      [-0.0299, -0.0405, -0.0159,  ...,  0.0400, -0.0095,  0.0218],\n",
              "                      [-0.0150,  0.0074,  0.0310,  ...,  0.0185, -0.0045,  0.0455],\n",
              "                      ...,\n",
              "                      [-0.0243, -0.0459,  0.0273,  ...,  0.0135,  0.0050, -0.0511],\n",
              "                      [ 0.0487,  0.0087,  0.0198,  ...,  0.0491, -0.0293, -0.0021],\n",
              "                      [-0.0217,  0.0138, -0.0380,  ...,  0.0134,  0.0222,  0.0189]])),\n",
              "             ('encoder.layers.6.self_attn.linears.3.bias',\n",
              "              tensor([-0.0149,  0.0087,  0.0257,  ...,  0.0223, -0.0121,  0.0040])),\n",
              "             ('encoder.layers.6.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0329,  0.0404, -0.0101,  ..., -0.0039, -0.0074,  0.0453],\n",
              "                      [-0.0591, -0.0495, -0.0466,  ...,  0.0351, -0.0533, -0.0234],\n",
              "                      [-0.0187, -0.0199, -0.0599,  ..., -0.0121,  0.0566,  0.0021],\n",
              "                      ...,\n",
              "                      [ 0.0436,  0.0583,  0.0356,  ..., -0.0714, -0.0562, -0.0534],\n",
              "                      [-0.0141, -0.0337,  0.0341,  ..., -0.0204,  0.0118, -0.0324],\n",
              "                      [-0.0151,  0.0131,  0.0071,  ...,  0.0571, -0.0268,  0.0173]])),\n",
              "             ('encoder.layers.6.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0048, -0.0273,  0.0238,  ...,  0.0229, -0.0181,  0.0007])),\n",
              "             ('encoder.layers.6.sublayer.0.norm.a_2',\n",
              "              tensor([1.0011, 0.9885, 1.0049,  ..., 0.9991, 0.9984, 0.9957])),\n",
              "             ('encoder.layers.6.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0008, -0.0027, -0.0016,  ...,  0.0006,  0.0003, -0.0055])),\n",
              "             ('encoder.layers.6.sublayer.1.norm.a_2',\n",
              "              tensor([1.0006, 1.0176, 1.0142,  ..., 1.0187, 1.0090, 1.0200])),\n",
              "             ('encoder.layers.6.sublayer.1.norm.b_2',\n",
              "              tensor([ 0.0022, -0.0009,  0.0005,  ..., -0.0009, -0.0017, -0.0109])),\n",
              "             ('encoder.layers.7.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0415, -0.0502,  0.0314,  ..., -0.0246,  0.0319, -0.0419],\n",
              "                      [ 0.0004, -0.0151,  0.0027,  ...,  0.0330, -0.0508,  0.0065],\n",
              "                      [ 0.0266, -0.0052,  0.0128,  ..., -0.0530,  0.0310,  0.0229],\n",
              "                      ...,\n",
              "                      [ 0.0434, -0.0383,  0.0128,  ..., -0.0344,  0.0485,  0.0355],\n",
              "                      [ 0.0281,  0.0099, -0.0159,  ..., -0.0047, -0.0039, -0.0525],\n",
              "                      [ 0.0062,  0.0582, -0.0270,  ...,  0.0201, -0.0454,  0.0029]])),\n",
              "             ('encoder.layers.7.self_attn.linears.0.bias',\n",
              "              tensor([-0.0176,  0.0078,  0.0207,  ...,  0.0231, -0.0180,  0.0009])),\n",
              "             ('encoder.layers.7.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0205,  0.0258, -0.0591,  ...,  0.0372,  0.0489, -0.0174],\n",
              "                      [-0.0140, -0.0086, -0.0219,  ..., -0.0418, -0.0116,  0.0128],\n",
              "                      [-0.0163, -0.0369, -0.0019,  ...,  0.0043, -0.0417,  0.0194],\n",
              "                      ...,\n",
              "                      [ 0.0168,  0.0027,  0.0067,  ...,  0.0372,  0.0201,  0.0168],\n",
              "                      [-0.0040,  0.0020,  0.0078,  ...,  0.0070, -0.0408,  0.0410],\n",
              "                      [ 0.0233,  0.0181,  0.0289,  ..., -0.0459, -0.0369,  0.0109]])),\n",
              "             ('encoder.layers.7.self_attn.linears.1.bias',\n",
              "              tensor([-1.7002e-02,  8.6146e-03,  2.6473e-02,  ...,  2.1091e-02,\n",
              "                      -1.1790e-02, -9.4817e-05])),\n",
              "             ('encoder.layers.7.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0274, -0.0117,  0.0156,  ..., -0.0021, -0.0383,  0.0525],\n",
              "                      [-0.0238,  0.0236,  0.0402,  ..., -0.0225, -0.0038,  0.0456],\n",
              "                      [-0.0248, -0.0096, -0.0083,  ...,  0.0400, -0.0035, -0.0113],\n",
              "                      ...,\n",
              "                      [ 0.0236,  0.0268,  0.0111,  ..., -0.0040,  0.0215, -0.0047],\n",
              "                      [ 0.0521,  0.0311, -0.0321,  ...,  0.0266, -0.0462,  0.0259],\n",
              "                      [ 0.0511,  0.0074, -0.0036,  ...,  0.0196, -0.0540, -0.0315]])),\n",
              "             ('encoder.layers.7.self_attn.linears.2.bias',\n",
              "              tensor([-0.0211,  0.0076,  0.0266,  ...,  0.0197, -0.0133,  0.0024])),\n",
              "             ('encoder.layers.7.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0529, -0.0472,  0.0083,  ...,  0.0468, -0.0560,  0.0100],\n",
              "                      [-0.0357, -0.0032, -0.0371,  ..., -0.0072, -0.0360,  0.0309],\n",
              "                      [-0.0541,  0.0248,  0.0366,  ..., -0.0142,  0.0107,  0.0064],\n",
              "                      ...,\n",
              "                      [-0.0342,  0.0100,  0.0596,  ...,  0.0302,  0.0543,  0.0160],\n",
              "                      [-0.0296, -0.0073,  0.0507,  ..., -0.0433, -0.0249,  0.0418],\n",
              "                      [ 0.0404, -0.0514, -0.0350,  ..., -0.0391,  0.0358,  0.0388]])),\n",
              "             ('encoder.layers.7.self_attn.linears.3.bias',\n",
              "              tensor([-0.0212,  0.0110,  0.0190,  ...,  0.0253, -0.0120, -0.0002])),\n",
              "             ('encoder.layers.7.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0318,  0.0171,  0.0198,  ..., -0.0306,  0.0412,  0.0345],\n",
              "                      [ 0.0067, -0.0550, -0.0283,  ..., -0.0234, -0.0186,  0.0460],\n",
              "                      [-0.0111, -0.0411, -0.0221,  ..., -0.0421,  0.0293, -0.0049],\n",
              "                      ...,\n",
              "                      [-0.0244, -0.0114, -0.0253,  ..., -0.0265,  0.0360, -0.0064],\n",
              "                      [ 0.0487,  0.0088,  0.0104,  ..., -0.0606, -0.0323,  0.0488],\n",
              "                      [ 0.0047,  0.0499, -0.0618,  ...,  0.0134,  0.0307, -0.0565]])),\n",
              "             ('encoder.layers.7.feed_forward.linears.0.bias',\n",
              "              tensor([-0.0090, -0.0247,  0.0237,  ...,  0.0224, -0.0201, -0.0002])),\n",
              "             ('encoder.layers.7.sublayer.0.norm.a_2',\n",
              "              tensor([1.0016, 0.9854, 1.0058,  ..., 1.0033, 1.0080, 1.0003])),\n",
              "             ('encoder.layers.7.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0048,  0.0005,  0.0139,  ..., -0.0018,  0.0017,  0.0095])),\n",
              "             ('encoder.layers.7.sublayer.1.norm.a_2',\n",
              "              tensor([1.0213, 1.0487, 1.0298,  ..., 1.0221, 1.0218, 1.0294])),\n",
              "             ('encoder.layers.7.sublayer.1.norm.b_2',\n",
              "              tensor([-0.0016, -0.0061, -0.0017,  ...,  0.0038, -0.0018,  0.0005])),\n",
              "             ('encoder.norm.a_2',\n",
              "              tensor([1.0593, 1.0204, 1.0305,  ..., 1.0185, 1.0413, 1.0205])),\n",
              "             ('encoder.norm.b_2',\n",
              "              tensor([-0.0394,  0.0058, -0.0052,  ..., -0.0009,  0.0279,  0.0139])),\n",
              "             ('src_embed.lut.weight',\n",
              "              tensor([[-0.0212,  0.0483, -0.0148,  ..., -0.0377, -0.0621,  0.0726],\n",
              "                      [ 0.0341, -0.0600, -0.0083,  ...,  0.0175,  0.0323,  0.0712],\n",
              "                      [ 0.0744,  0.0237, -0.0668,  ...,  0.0729, -0.0232,  0.0124],\n",
              "                      ...,\n",
              "                      [ 0.0659, -0.0425,  0.0024,  ..., -0.0373, -0.0127,  0.0401],\n",
              "                      [-0.0683, -0.0583,  0.0357,  ...,  0.0517, -0.0268, -0.0253],\n",
              "                      [-0.0371, -0.0061,  0.0745,  ..., -0.0188,  0.0306, -0.0243]])),\n",
              "             ('src_embed.lut.bias',\n",
              "              tensor([-0.1589,  0.1621, -0.1299,  ...,  0.0922,  0.1594, -0.1628]))])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YHtt4w-OwWmQ"
      },
      "source": [
        "\n",
        "def tryi():\n",
        "  model_state_dict = model1.state_dict()\n",
        "  for name, param in pretrained_state_dict.items():\n",
        "      \n",
        "      if isinstance(param, torch.nn.Parameter):\n",
        "          param = param.data\n",
        "      model_state_dict[name].copy_(param)\n",
        "      if 'generator' in name:\n",
        "           return"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zpp4iUaT28nh",
        "outputId": "61472b9f-30b5-459a-cee0-eb5c7536c8d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model_state_dict"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "OrderedDict([('encoder.layers.0.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0523,  0.0197, -0.0003,  ...,  0.0171,  0.0427, -0.0305],\n",
              "                      [ 0.0014,  0.0492, -0.0425,  ...,  0.0028, -0.0104,  0.0015],\n",
              "                      [ 0.0403,  0.0453,  0.0339,  ...,  0.0180, -0.0286, -0.0413],\n",
              "                      ...,\n",
              "                      [ 0.0378,  0.0147,  0.0352,  ...,  0.0411, -0.0403,  0.0409],\n",
              "                      [ 0.0477,  0.0430,  0.0343,  ...,  0.0194,  0.0390, -0.0340],\n",
              "                      [-0.0023, -0.0492, -0.0140,  ..., -0.0462,  0.0443, -0.0047]])),\n",
              "             ('encoder.layers.0.self_attn.linears.0.bias',\n",
              "              tensor([-0.0189,  0.0018,  0.0261,  ...,  0.0225, -0.0111, -0.0002])),\n",
              "             ('encoder.layers.0.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0003, -0.0490, -0.0487,  ...,  0.0246,  0.0356, -0.0149],\n",
              "                      [ 0.0270,  0.0260, -0.0352,  ...,  0.0345, -0.0443, -0.0432],\n",
              "                      [-0.0615,  0.0374, -0.0264,  ...,  0.0177,  0.0370, -0.0379],\n",
              "                      ...,\n",
              "                      [-0.0119, -0.0468, -0.0118,  ...,  0.0114,  0.0360, -0.0005],\n",
              "                      [ 0.0275,  0.0526, -0.0453,  ..., -0.0517, -0.0019, -0.0282],\n",
              "                      [-0.0230, -0.0299,  0.0233,  ..., -0.0159, -0.0045, -0.0058]])),\n",
              "             ('encoder.layers.0.self_attn.linears.1.bias',\n",
              "              tensor([-0.0168,  0.0088,  0.0264,  ...,  0.0202, -0.0113,  0.0006])),\n",
              "             ('encoder.layers.0.self_attn.linears.2.weight',\n",
              "              tensor([[ 0.0513, -0.0106, -0.0412,  ...,  0.0180, -0.0074,  0.0028],\n",
              "                      [-0.0009, -0.0138, -0.0422,  ..., -0.0180, -0.0037, -0.0056],\n",
              "                      [-0.0270,  0.0103, -0.0265,  ...,  0.0260,  0.0143,  0.0044],\n",
              "                      ...,\n",
              "                      [ 0.0060,  0.0267,  0.0219,  ...,  0.0316, -0.0099,  0.0610],\n",
              "                      [ 0.0131,  0.0128,  0.0246,  ..., -0.0120, -0.0461, -0.0189],\n",
              "                      [-0.0096, -0.0381,  0.0436,  ..., -0.0226,  0.0010, -0.0450]])),\n",
              "             ('encoder.layers.0.self_attn.linears.2.bias',\n",
              "              tensor([-0.0160,  0.0083,  0.0266,  ...,  0.0216, -0.0120,  0.0019])),\n",
              "             ('encoder.layers.0.self_attn.linears.3.weight',\n",
              "              tensor([[-0.0560, -0.0033,  0.0278,  ..., -0.0211,  0.0405,  0.0467],\n",
              "                      [-0.0049,  0.0341, -0.0418,  ...,  0.0566, -0.0022, -0.0452],\n",
              "                      [ 0.0313,  0.0056,  0.0162,  ...,  0.0435, -0.0417,  0.0092],\n",
              "                      ...,\n",
              "                      [-0.0346, -0.0077, -0.0521,  ..., -0.0144,  0.0439, -0.0430],\n",
              "                      [-0.0262, -0.0225, -0.0285,  ..., -0.0320,  0.0212,  0.0063],\n",
              "                      [ 0.0445,  0.0331, -0.0143,  ...,  0.0089,  0.0076, -0.0104]])),\n",
              "             ('encoder.layers.0.self_attn.linears.3.bias',\n",
              "              tensor([-0.0188,  0.0086,  0.0284,  ...,  0.0211, -0.0092, -0.0046])),\n",
              "             ('encoder.layers.0.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0506, -0.0421, -0.0479,  ..., -0.0197, -0.0440, -0.0083],\n",
              "                      [-0.0126, -0.0443, -0.0563,  ..., -0.0034,  0.0641,  0.0497],\n",
              "                      [-0.0234, -0.0294, -0.0069,  ..., -0.0423, -0.0396, -0.0260],\n",
              "                      ...,\n",
              "                      [-0.0162, -0.0506, -0.0119,  ...,  0.0080,  0.0319, -0.0441],\n",
              "                      [-0.0266,  0.0057, -0.0084,  ..., -0.0529, -0.0490,  0.0214],\n",
              "                      [-0.0172, -0.0597, -0.0315,  ..., -0.0059, -0.0340,  0.0187]])),\n",
              "             ('encoder.layers.0.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0004, -0.0240,  0.0229,  ...,  0.0229, -0.0180, -0.0021])),\n",
              "             ('encoder.layers.0.sublayer.0.norm.a_2',\n",
              "              tensor([0.9977, 0.9980, 1.0044,  ..., 0.9923, 0.9925, 1.0037])),\n",
              "             ('encoder.layers.0.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0014, -0.0046, -0.0007,  ..., -0.0086, -0.0035,  0.0059])),\n",
              "             ('encoder.layers.0.sublayer.1.norm.a_2',\n",
              "              tensor([1.0061, 1.0072, 0.9989,  ..., 1.0021, 1.0076, 1.0108])),\n",
              "             ('encoder.layers.0.sublayer.1.norm.b_2',\n",
              "              tensor([-3.5557e-04,  2.7526e-03,  2.9475e-03,  ..., -3.4370e-03,\n",
              "                       9.6726e-05, -6.6116e-03])),\n",
              "             ('encoder.layers.1.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0058,  0.0317, -0.0087,  ...,  0.0059, -0.0506, -0.0272],\n",
              "                      [ 0.0048, -0.0072, -0.0316,  ...,  0.0263,  0.0309, -0.0445],\n",
              "                      [-0.0364, -0.0046,  0.0322,  ..., -0.0404,  0.0353, -0.0544],\n",
              "                      ...,\n",
              "                      [ 0.0172,  0.0094, -0.0479,  ..., -0.0341, -0.0320, -0.0480],\n",
              "                      [-0.0272, -0.0082, -0.0083,  ...,  0.0484,  0.0451, -0.0088],\n",
              "                      [ 0.0296, -0.0205, -0.0135,  ...,  0.0535, -0.0251,  0.0442]])),\n",
              "             ('encoder.layers.1.self_attn.linears.0.bias',\n",
              "              tensor([-0.0224,  0.0210,  0.0330,  ...,  0.0189, -0.0095, -0.0153])),\n",
              "             ('encoder.layers.1.self_attn.linears.1.weight',\n",
              "              tensor([[ 0.0413,  0.0533, -0.0137,  ...,  0.0170,  0.0414,  0.0217],\n",
              "                      [ 0.0226,  0.0157,  0.0269,  ..., -0.0220,  0.0043,  0.0236],\n",
              "                      [ 0.0400, -0.0283,  0.0573,  ..., -0.0514, -0.0143, -0.0349],\n",
              "                      ...,\n",
              "                      [ 0.0145, -0.0113, -0.0299,  ..., -0.0214,  0.0281, -0.0521],\n",
              "                      [ 0.0049,  0.0177, -0.0181,  ..., -0.0077,  0.0348, -0.0238],\n",
              "                      [-0.0076, -0.0046,  0.0186,  ..., -0.0428, -0.0180,  0.0013]])),\n",
              "             ('encoder.layers.1.self_attn.linears.1.bias',\n",
              "              tensor([-1.6915e-02,  8.6603e-03,  2.6303e-02,  ...,  2.1149e-02,\n",
              "                      -1.1799e-02, -9.4858e-05])),\n",
              "             ('encoder.layers.1.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0081, -0.0458, -0.0108,  ...,  0.0074,  0.0480,  0.0375],\n",
              "                      [ 0.0047, -0.0504, -0.0232,  ..., -0.0167, -0.0065, -0.0195],\n",
              "                      [-0.0011, -0.0429, -0.0443,  ..., -0.0282, -0.0341, -0.0483],\n",
              "                      ...,\n",
              "                      [ 0.0134, -0.0006,  0.0407,  ..., -0.0464,  0.0212, -0.0308],\n",
              "                      [-0.0286,  0.0223,  0.0097,  ...,  0.0165,  0.0002,  0.0111],\n",
              "                      [ 0.0373,  0.0267, -0.0069,  ..., -0.0259, -0.0409, -0.0451]])),\n",
              "             ('encoder.layers.1.self_attn.linears.2.bias',\n",
              "              tensor([-0.0123,  0.0073,  0.0237,  ...,  0.0215, -0.0097,  0.0022])),\n",
              "             ('encoder.layers.1.self_attn.linears.3.weight',\n",
              "              tensor([[-4.6729e-02, -4.0582e-02, -1.2572e-02,  ..., -7.3779e-03,\n",
              "                       -3.6672e-03, -4.6826e-02],\n",
              "                      [ 4.7870e-02, -3.9885e-02, -5.2955e-03,  ...,  1.6887e-02,\n",
              "                       -6.5864e-03, -1.0508e-02],\n",
              "                      [-2.1034e-02,  4.6441e-02,  4.1189e-02,  ..., -2.4511e-02,\n",
              "                       -1.7949e-02,  1.6522e-02],\n",
              "                      ...,\n",
              "                      [ 2.7128e-02,  3.3623e-02, -2.9111e-03,  ...,  4.1705e-02,\n",
              "                        4.5360e-02, -4.5868e-02],\n",
              "                      [ 5.2379e-02,  2.5552e-02,  2.7708e-02,  ..., -1.8306e-02,\n",
              "                       -6.3889e-06, -2.0187e-02],\n",
              "                      [-4.0883e-02,  3.6099e-02, -4.6619e-02,  ...,  3.2978e-02,\n",
              "                       -3.8937e-02, -9.4912e-04]])),\n",
              "             ('encoder.layers.1.self_attn.linears.3.bias',\n",
              "              tensor([-0.0164,  0.0076,  0.0263,  ...,  0.0231, -0.0089, -0.0002])),\n",
              "             ('encoder.layers.1.feed_forward.linears.0.weight',\n",
              "              tensor([[ 0.0083, -0.0142, -0.0514,  ..., -0.0187, -0.0126, -0.0228],\n",
              "                      [-0.0080, -0.0144,  0.0443,  ..., -0.0397,  0.0376,  0.0120],\n",
              "                      [-0.0422, -0.0519,  0.0174,  ...,  0.0249,  0.0363,  0.0388],\n",
              "                      ...,\n",
              "                      [-0.0145, -0.0225,  0.0291,  ..., -0.0278,  0.0106,  0.0086],\n",
              "                      [ 0.0054, -0.0045,  0.0353,  ..., -0.0562, -0.0349, -0.0447],\n",
              "                      [-0.0239, -0.0428,  0.0378,  ...,  0.0260, -0.0237,  0.0414]])),\n",
              "             ('encoder.layers.1.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0046, -0.0287,  0.0204,  ...,  0.0235, -0.0198, -0.0013])),\n",
              "             ('encoder.layers.1.sublayer.0.norm.a_2',\n",
              "              tensor([0.9919, 0.9947, 1.0000,  ..., 0.9882, 0.9950, 1.0061])),\n",
              "             ('encoder.layers.1.sublayer.0.norm.b_2',\n",
              "              tensor([-0.0050, -0.0032, -0.0024,  ...,  0.0022,  0.0028, -0.0025])),\n",
              "             ('encoder.layers.1.sublayer.1.norm.a_2',\n",
              "              tensor([0.9985, 1.0121, 1.0007,  ..., 0.9999, 1.0065, 1.0168])),\n",
              "             ('encoder.layers.1.sublayer.1.norm.b_2',\n",
              "              tensor([-2.8897e-03, -9.1054e-05,  1.4950e-03,  ..., -1.4927e-03,\n",
              "                       4.7737e-03,  1.8817e-03])),\n",
              "             ('encoder.layers.2.self_attn.linears.0.weight',\n",
              "              tensor([[-0.0101, -0.0493,  0.0050,  ..., -0.0254, -0.0141,  0.0204],\n",
              "                      [-0.0189, -0.0531,  0.0157,  ...,  0.0378,  0.0156,  0.0625],\n",
              "                      [-0.0169, -0.0268, -0.0404,  ..., -0.0152,  0.0473, -0.0178],\n",
              "                      ...,\n",
              "                      [-0.0400, -0.0446,  0.0154,  ..., -0.0317,  0.0412,  0.0367],\n",
              "                      [ 0.0076,  0.0159,  0.0042,  ..., -0.0373,  0.0235,  0.0490],\n",
              "                      [-0.0350,  0.0447, -0.0222,  ..., -0.0184,  0.0293,  0.0004]])),\n",
              "             ('encoder.layers.2.self_attn.linears.0.bias',\n",
              "              tensor([-0.0314, -0.0093,  0.0136,  ...,  0.0193, -0.0179, -0.0004])),\n",
              "             ('encoder.layers.2.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0079,  0.0436,  0.0320,  ...,  0.0028,  0.0585, -0.0308],\n",
              "                      [ 0.0266,  0.0031, -0.0612,  ..., -0.0623, -0.0326, -0.0166],\n",
              "                      [-0.0008,  0.0163, -0.0268,  ...,  0.0260,  0.0578, -0.0117],\n",
              "                      ...,\n",
              "                      [-0.0166, -0.0274, -0.0531,  ...,  0.0384,  0.0221, -0.0404],\n",
              "                      [-0.0414, -0.0340,  0.0337,  ...,  0.0532,  0.0399, -0.0166],\n",
              "                      [-0.0315,  0.0051,  0.0113,  ..., -0.0418, -0.0366,  0.0497]])),\n",
              "             ('encoder.layers.2.self_attn.linears.1.bias',\n",
              "              tensor([-0.0169,  0.0087,  0.0265,  ...,  0.0211, -0.0118, -0.0002])),\n",
              "             ('encoder.layers.2.self_attn.linears.2.weight',\n",
              "              tensor([[ 0.0368,  0.0275,  0.0282,  ...,  0.0029,  0.0603,  0.0355],\n",
              "                      [-0.0415,  0.0477, -0.0363,  ..., -0.0465,  0.0457, -0.0163],\n",
              "                      [-0.0243, -0.0168,  0.0235,  ...,  0.0148, -0.0163, -0.0112],\n",
              "                      ...,\n",
              "                      [-0.0502,  0.0435, -0.0503,  ...,  0.0321, -0.0273, -0.0458],\n",
              "                      [ 0.0158,  0.0221,  0.0204,  ...,  0.0038,  0.0186, -0.0210],\n",
              "                      [-0.0380, -0.0383, -0.0075,  ...,  0.0260,  0.0184, -0.0050]])),\n",
              "             ('encoder.layers.2.self_attn.linears.2.bias',\n",
              "              tensor([-0.0138,  0.0119,  0.0270,  ...,  0.0208, -0.0120, -0.0037])),\n",
              "             ('encoder.layers.2.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0048, -0.0397,  0.0189,  ...,  0.0018,  0.0195,  0.0059],\n",
              "                      [ 0.0332, -0.0151, -0.0172,  ...,  0.0238,  0.0137,  0.0065],\n",
              "                      [ 0.0109, -0.0193, -0.0578,  ...,  0.0315, -0.0372,  0.0433],\n",
              "                      ...,\n",
              "                      [-0.0547,  0.0438,  0.0118,  ..., -0.0535,  0.0132, -0.0280],\n",
              "                      [-0.0134, -0.0481,  0.0198,  ...,  0.0139, -0.0144, -0.0419],\n",
              "                      [ 0.0044,  0.0357, -0.0154,  ..., -0.0409, -0.0140,  0.0077]])),\n",
              "             ('encoder.layers.2.self_attn.linears.3.bias',\n",
              "              tensor([-0.0171,  0.0069,  0.0249,  ...,  0.0264, -0.0093, -0.0006])),\n",
              "             ('encoder.layers.2.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0218, -0.0095,  0.0272,  ..., -0.0309,  0.0162, -0.0091],\n",
              "                      [ 0.0455, -0.0561, -0.0350,  ...,  0.0010,  0.0445, -0.0212],\n",
              "                      [ 0.0059,  0.0497,  0.0022,  ...,  0.0441,  0.0211,  0.0321],\n",
              "                      ...,\n",
              "                      [ 0.0254,  0.0460, -0.0509,  ..., -0.0237,  0.0442,  0.0248],\n",
              "                      [-0.0275,  0.0378,  0.0099,  ..., -0.0520, -0.0496, -0.0066],\n",
              "                      [-0.0134, -0.0217, -0.0475,  ..., -0.0116, -0.0012, -0.0275]])),\n",
              "             ('encoder.layers.2.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0002, -0.0322,  0.0231,  ...,  0.0239, -0.0200, -0.0045])),\n",
              "             ('encoder.layers.2.sublayer.0.norm.a_2',\n",
              "              tensor([1.0012, 0.9998, 1.0066,  ..., 1.0013, 1.0034, 0.9976])),\n",
              "             ('encoder.layers.2.sublayer.0.norm.b_2',\n",
              "              tensor([-2.0655e-04, -1.3716e-03, -2.1849e-03,  ..., -2.7905e-03,\n",
              "                       1.6901e-03, -4.1606e-05])),\n",
              "             ('encoder.layers.2.sublayer.1.norm.a_2',\n",
              "              tensor([1.0017, 1.0030, 1.0042,  ..., 1.0084, 1.0054, 1.0083])),\n",
              "             ('encoder.layers.2.sublayer.1.norm.b_2',\n",
              "              tensor([ 3.0622e-03, -3.7244e-03, -3.9762e-03,  ...,  3.5263e-05,\n",
              "                       1.2276e-03,  8.5568e-04])),\n",
              "             ('encoder.layers.3.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0319,  0.0499, -0.0031,  ...,  0.0044, -0.0461,  0.0397],\n",
              "                      [ 0.0397,  0.0355,  0.0414,  ..., -0.0445,  0.0021, -0.0440],\n",
              "                      [-0.0583, -0.0248,  0.0236,  ..., -0.0073, -0.0348,  0.0017],\n",
              "                      ...,\n",
              "                      [ 0.0326, -0.0527, -0.0175,  ..., -0.0128,  0.0046, -0.0341],\n",
              "                      [-0.0129, -0.0490, -0.0429,  ...,  0.0054,  0.0394,  0.0250],\n",
              "                      [-0.0364,  0.0059, -0.0284,  ...,  0.0323, -0.0212,  0.0112]])),\n",
              "             ('encoder.layers.3.self_attn.linears.0.bias',\n",
              "              tensor([-0.0145,  0.0141,  0.0298,  ...,  0.0135, -0.0126, -0.0152])),\n",
              "             ('encoder.layers.3.self_attn.linears.1.weight',\n",
              "              tensor([[ 0.0215, -0.0113, -0.0088,  ..., -0.0442,  0.0532,  0.0529],\n",
              "                      [-0.0259, -0.0006, -0.0051,  ...,  0.0455,  0.0390, -0.0011],\n",
              "                      [ 0.0380, -0.0447,  0.0308,  ...,  0.0332, -0.0350,  0.0312],\n",
              "                      ...,\n",
              "                      [-0.0158,  0.0258, -0.0260,  ..., -0.0259,  0.0269, -0.0299],\n",
              "                      [ 0.0416, -0.0022,  0.0146,  ..., -0.0509,  0.0307, -0.0451],\n",
              "                      [ 0.0449, -0.0019,  0.0014,  ..., -0.0621, -0.0162,  0.0481]])),\n",
              "             ('encoder.layers.3.self_attn.linears.1.bias',\n",
              "              tensor([-0.0169,  0.0087,  0.0264,  ...,  0.0210, -0.0118, -0.0002])),\n",
              "             ('encoder.layers.3.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0069, -0.0098,  0.0224,  ..., -0.0018, -0.0512,  0.0456],\n",
              "                      [ 0.0167, -0.0118, -0.0542,  ...,  0.0216,  0.0075,  0.0303],\n",
              "                      [-0.0127,  0.0332, -0.0273,  ..., -0.0328,  0.0388, -0.0019],\n",
              "                      ...,\n",
              "                      [ 0.0016, -0.0227,  0.0455,  ..., -0.0240, -0.0488, -0.0231],\n",
              "                      [ 0.0467, -0.0070, -0.0386,  ...,  0.0251, -0.0013, -0.0489],\n",
              "                      [ 0.0109, -0.0098, -0.0062,  ..., -0.0381, -0.0083,  0.0272]])),\n",
              "             ('encoder.layers.3.self_attn.linears.2.bias',\n",
              "              tensor([-0.0204,  0.0079,  0.0243,  ...,  0.0234, -0.0111,  0.0021])),\n",
              "             ('encoder.layers.3.self_attn.linears.3.weight',\n",
              "              tensor([[-0.0039,  0.0293, -0.0227,  ...,  0.0502,  0.0071, -0.0070],\n",
              "                      [-0.0297, -0.0242,  0.0135,  ...,  0.0502,  0.0495,  0.0204],\n",
              "                      [-0.0581,  0.0398, -0.0397,  ..., -0.0377,  0.0286, -0.0321],\n",
              "                      ...,\n",
              "                      [ 0.0065,  0.0536,  0.0358,  ...,  0.0421, -0.0244, -0.0213],\n",
              "                      [ 0.0413,  0.0450, -0.0432,  ..., -0.0189, -0.0399,  0.0052],\n",
              "                      [ 0.0566,  0.0295, -0.0447,  ..., -0.0404, -0.0407, -0.0051]])),\n",
              "             ('encoder.layers.3.self_attn.linears.3.bias',\n",
              "              tensor([-0.0168,  0.0057,  0.0244,  ...,  0.0241, -0.0118, -0.0002])),\n",
              "             ('encoder.layers.3.feed_forward.linears.0.weight',\n",
              "              tensor([[ 0.0058,  0.0258, -0.0403,  ..., -0.0274,  0.0353, -0.0139],\n",
              "                      [ 0.0098,  0.0245,  0.0188,  ..., -0.0299,  0.0292,  0.0160],\n",
              "                      [ 0.0409, -0.0099,  0.0306,  ..., -0.0511, -0.0462, -0.0615],\n",
              "                      ...,\n",
              "                      [ 0.0084, -0.0595, -0.0301,  ...,  0.0046, -0.0028,  0.0490],\n",
              "                      [ 0.0321,  0.0437, -0.0159,  ...,  0.0246, -0.0632,  0.0463],\n",
              "                      [-0.0024,  0.0072, -0.0487,  ...,  0.0658, -0.0234, -0.0159]])),\n",
              "             ('encoder.layers.3.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0029, -0.0293,  0.0250,  ...,  0.0202, -0.0225,  0.0026])),\n",
              "             ('encoder.layers.3.sublayer.0.norm.a_2',\n",
              "              tensor([0.9957, 0.9973, 1.0031,  ..., 1.0008, 1.0010, 1.0012])),\n",
              "             ('encoder.layers.3.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0031, -0.0004, -0.0051,  ..., -0.0002,  0.0020, -0.0013])),\n",
              "             ('encoder.layers.3.sublayer.1.norm.a_2',\n",
              "              tensor([1.0045, 1.0125, 0.9995,  ..., 1.0043, 1.0264, 1.0087])),\n",
              "             ('encoder.layers.3.sublayer.1.norm.b_2',\n",
              "              tensor([-0.0023,  0.0047, -0.0056,  ...,  0.0019,  0.0032, -0.0043])),\n",
              "             ('encoder.layers.4.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0253,  0.0171, -0.0514,  ..., -0.0209,  0.0183,  0.0210],\n",
              "                      [ 0.0171, -0.0219,  0.0169,  ...,  0.0272, -0.0446, -0.0577],\n",
              "                      [-0.0318,  0.0403,  0.0295,  ...,  0.0184,  0.0276, -0.0614],\n",
              "                      ...,\n",
              "                      [ 0.0439,  0.0350, -0.0427,  ...,  0.0010,  0.0060,  0.0200],\n",
              "                      [ 0.0029,  0.0153,  0.0342,  ...,  0.0242,  0.0393, -0.0020],\n",
              "                      [-0.0227,  0.0438,  0.0516,  ..., -0.0276, -0.0528,  0.0308]])),\n",
              "             ('encoder.layers.4.self_attn.linears.0.bias',\n",
              "              tensor([-0.0156,  0.0064,  0.0204,  ...,  0.0234, -0.0103,  0.0045])),\n",
              "             ('encoder.layers.4.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0288,  0.0543,  0.0070,  ...,  0.0349,  0.0072, -0.0102],\n",
              "                      [-0.0500,  0.0073, -0.0156,  ...,  0.0365, -0.0142,  0.0250],\n",
              "                      [-0.0012,  0.0533,  0.0042,  ..., -0.0422, -0.0356,  0.0050],\n",
              "                      ...,\n",
              "                      [ 0.0197,  0.0074,  0.0445,  ..., -0.0508, -0.0342, -0.0035],\n",
              "                      [-0.0494,  0.0441,  0.0169,  ...,  0.0384,  0.0268, -0.0085],\n",
              "                      [ 0.0126,  0.0400, -0.0461,  ...,  0.0330, -0.0261,  0.0070]])),\n",
              "             ('encoder.layers.4.self_attn.linears.1.bias',\n",
              "              tensor([-1.7398e-02,  9.2496e-03,  2.6399e-02,  ...,  2.1093e-02,\n",
              "                      -1.1747e-02, -7.2788e-05])),\n",
              "             ('encoder.layers.4.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0083,  0.0324, -0.0212,  ..., -0.0515,  0.0332, -0.0226],\n",
              "                      [-0.0149,  0.0210, -0.0424,  ...,  0.0415, -0.0326, -0.0116],\n",
              "                      [ 0.0047, -0.0218, -0.0141,  ..., -0.0086, -0.0041, -0.0294],\n",
              "                      ...,\n",
              "                      [-0.0279,  0.0224, -0.0048,  ..., -0.0038,  0.0201,  0.0550],\n",
              "                      [-0.0148,  0.0468,  0.0362,  ..., -0.0434, -0.0404, -0.0323],\n",
              "                      [ 0.0308, -0.0308, -0.0412,  ...,  0.0235, -0.0410,  0.0057]])),\n",
              "             ('encoder.layers.4.self_attn.linears.2.bias',\n",
              "              tensor([-0.0190,  0.0078,  0.0272,  ...,  0.0240, -0.0166, -0.0009])),\n",
              "             ('encoder.layers.4.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0367, -0.0441, -0.0380,  ...,  0.0201, -0.0433, -0.0020],\n",
              "                      [-0.0007,  0.0158, -0.0347,  ...,  0.0006,  0.0452,  0.0109],\n",
              "                      [-0.0512, -0.0328,  0.0350,  ..., -0.0377, -0.0366, -0.0514],\n",
              "                      ...,\n",
              "                      [-0.0309,  0.0495,  0.0260,  ..., -0.0406, -0.0505, -0.0103],\n",
              "                      [ 0.0358,  0.0385, -0.0174,  ..., -0.0459, -0.0351,  0.0157],\n",
              "                      [-0.0244,  0.0129,  0.0485,  ..., -0.0480, -0.0103,  0.0401]])),\n",
              "             ('encoder.layers.4.self_attn.linears.3.bias',\n",
              "              tensor([-0.0153,  0.0077,  0.0234,  ...,  0.0225, -0.0141,  0.0002])),\n",
              "             ('encoder.layers.4.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0209,  0.0098, -0.0379,  ...,  0.0503,  0.0028,  0.0306],\n",
              "                      [ 0.0213, -0.0210,  0.0236,  ...,  0.0233, -0.0043,  0.0170],\n",
              "                      [ 0.0322,  0.0399, -0.0390,  ...,  0.0025,  0.0069,  0.0238],\n",
              "                      ...,\n",
              "                      [ 0.0043, -0.0115,  0.0202,  ..., -0.0584, -0.0021, -0.0234],\n",
              "                      [-0.0103,  0.0375, -0.0282,  ...,  0.0285, -0.0110,  0.0079],\n",
              "                      [ 0.0409,  0.0146,  0.0218,  ..., -0.0338, -0.0311, -0.0454]])),\n",
              "             ('encoder.layers.4.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0076, -0.0279,  0.0205,  ...,  0.0182, -0.0240, -0.0023])),\n",
              "             ('encoder.layers.4.sublayer.0.norm.a_2',\n",
              "              tensor([0.9926, 0.9982, 1.0083,  ..., 1.0012, 0.9946, 1.0045])),\n",
              "             ('encoder.layers.4.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0003, -0.0025, -0.0015,  ...,  0.0017,  0.0001,  0.0034])),\n",
              "             ('encoder.layers.4.sublayer.1.norm.a_2',\n",
              "              tensor([1.0020, 1.0145, 0.9950,  ..., 1.0035, 1.0081, 1.0067])),\n",
              "             ('encoder.layers.4.sublayer.1.norm.b_2',\n",
              "              tensor([-0.0002, -0.0046, -0.0042,  ..., -0.0003, -0.0003,  0.0016])),\n",
              "             ('encoder.layers.5.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0027,  0.0502, -0.0187,  ...,  0.0299,  0.0314,  0.0078],\n",
              "                      [-0.0251,  0.0437, -0.0397,  ...,  0.0506, -0.0228, -0.0229],\n",
              "                      [ 0.0154,  0.0432,  0.0349,  ..., -0.0066, -0.0396, -0.0124],\n",
              "                      ...,\n",
              "                      [ 0.0091, -0.0253, -0.0474,  ...,  0.0588, -0.0450, -0.0334],\n",
              "                      [-0.0556, -0.0421, -0.0020,  ..., -0.0450,  0.0145, -0.0437],\n",
              "                      [-0.0217,  0.0089,  0.0304,  ...,  0.0197, -0.0511,  0.0352]])),\n",
              "             ('encoder.layers.5.self_attn.linears.0.bias',\n",
              "              tensor([-0.0098,  0.0174,  0.0225,  ...,  0.0143, -0.0189,  0.0054])),\n",
              "             ('encoder.layers.5.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0015, -0.0358, -0.0015,  ..., -0.0038,  0.0441, -0.0327],\n",
              "                      [ 0.0137, -0.0028, -0.0169,  ...,  0.0204,  0.0142,  0.0098],\n",
              "                      [ 0.0032, -0.0543,  0.0425,  ..., -0.0071,  0.0007,  0.0376],\n",
              "                      ...,\n",
              "                      [-0.0436,  0.0523,  0.0040,  ..., -0.0228, -0.0334, -0.0586],\n",
              "                      [ 0.0170,  0.0060, -0.0216,  ...,  0.0443,  0.0056,  0.0371],\n",
              "                      [ 0.0223,  0.0323,  0.0508,  ...,  0.0136,  0.0379,  0.0044]])),\n",
              "             ('encoder.layers.5.self_attn.linears.1.bias',\n",
              "              tensor([-0.0168,  0.0088,  0.0264,  ...,  0.0211, -0.0118, -0.0002])),\n",
              "             ('encoder.layers.5.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0425,  0.0460,  0.0385,  ..., -0.0148, -0.0283,  0.0217],\n",
              "                      [ 0.0533,  0.0393,  0.0475,  ..., -0.0485, -0.0464, -0.0103],\n",
              "                      [-0.0070,  0.0019, -0.0132,  ...,  0.0159,  0.0513,  0.0451],\n",
              "                      ...,\n",
              "                      [ 0.0388,  0.0434,  0.0128,  ...,  0.0197,  0.0207, -0.0462],\n",
              "                      [-0.0022, -0.0250,  0.0446,  ..., -0.0478,  0.0076,  0.0253],\n",
              "                      [-0.0069, -0.0084,  0.0059,  ...,  0.0200, -0.0180,  0.0260]])),\n",
              "             ('encoder.layers.5.self_attn.linears.2.bias',\n",
              "              tensor([-0.0152,  0.0127,  0.0281,  ...,  0.0209, -0.0128, -0.0005])),\n",
              "             ('encoder.layers.5.self_attn.linears.3.weight',\n",
              "              tensor([[-0.0060,  0.0208, -0.0342,  ..., -0.0338,  0.0132,  0.0479],\n",
              "                      [-0.0251, -0.0093,  0.0258,  ..., -0.0174,  0.0243,  0.0207],\n",
              "                      [ 0.0047,  0.0304,  0.0428,  ..., -0.0205,  0.0261,  0.0494],\n",
              "                      ...,\n",
              "                      [ 0.0035,  0.0259, -0.0148,  ..., -0.0066,  0.0107, -0.0110],\n",
              "                      [-0.0462,  0.0050,  0.0192,  ..., -0.0212, -0.0311, -0.0309],\n",
              "                      [ 0.0459, -0.0570,  0.0215,  ...,  0.0197,  0.0279,  0.0494]])),\n",
              "             ('encoder.layers.5.self_attn.linears.3.bias',\n",
              "              tensor([-0.0134,  0.0064,  0.0231,  ...,  0.0222, -0.0104,  0.0015])),\n",
              "             ('encoder.layers.5.feed_forward.linears.0.weight',\n",
              "              tensor([[ 0.0322, -0.0207,  0.0014,  ..., -0.0066, -0.0540, -0.0087],\n",
              "                      [-0.0287, -0.0260, -0.0004,  ..., -0.0117, -0.0348, -0.0507],\n",
              "                      [ 0.0149, -0.0258, -0.0524,  ...,  0.0166, -0.0213,  0.0055],\n",
              "                      ...,\n",
              "                      [-0.0036,  0.0255, -0.0065,  ...,  0.0190, -0.0334,  0.0291],\n",
              "                      [ 0.0301,  0.0567,  0.0051,  ...,  0.0027,  0.0168, -0.0396],\n",
              "                      [ 0.0073, -0.0377,  0.0456,  ...,  0.0027, -0.0189,  0.0249]])),\n",
              "             ('encoder.layers.5.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0015, -0.0295,  0.0203,  ...,  0.0217, -0.0208, -0.0018])),\n",
              "             ('encoder.layers.5.sublayer.0.norm.a_2',\n",
              "              tensor([0.9922, 0.9949, 1.0026,  ..., 0.9877, 0.9931, 1.0017])),\n",
              "             ('encoder.layers.5.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0007, -0.0020, -0.0016,  ...,  0.0021, -0.0066,  0.0013])),\n",
              "             ('encoder.layers.5.sublayer.1.norm.a_2',\n",
              "              tensor([1.0059, 1.0031, 1.0133,  ..., 1.0238, 1.0100, 1.0167])),\n",
              "             ('encoder.layers.5.sublayer.1.norm.b_2',\n",
              "              tensor([ 0.0040, -0.0036,  0.0013,  ..., -0.0009,  0.0002,  0.0028])),\n",
              "             ('encoder.layers.6.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0304,  0.0387, -0.0403,  ..., -0.0464,  0.0049, -0.0196],\n",
              "                      [ 0.0325, -0.0084,  0.0333,  ..., -0.0223, -0.0414,  0.0308],\n",
              "                      [ 0.0386, -0.0249,  0.0061,  ..., -0.0505, -0.0548, -0.0374],\n",
              "                      ...,\n",
              "                      [-0.0344,  0.0492, -0.0227,  ..., -0.0550,  0.0393,  0.0233],\n",
              "                      [ 0.0158, -0.0272,  0.0272,  ...,  0.0148, -0.0299, -0.0482],\n",
              "                      [-0.0181, -0.0019,  0.0395,  ..., -0.0365,  0.0215,  0.0212]])),\n",
              "             ('encoder.layers.6.self_attn.linears.0.bias',\n",
              "              tensor([-0.0152,  0.0108,  0.0250,  ...,  0.0174, -0.0187, -0.0027])),\n",
              "             ('encoder.layers.6.self_attn.linears.1.weight',\n",
              "              tensor([[ 0.0475,  0.0228, -0.0018,  ...,  0.0282,  0.0225,  0.0517],\n",
              "                      [-0.0002, -0.0437,  0.0307,  ...,  0.0288, -0.0458,  0.0360],\n",
              "                      [ 0.0149,  0.0358,  0.0313,  ..., -0.0012, -0.0137, -0.0401],\n",
              "                      ...,\n",
              "                      [ 0.0468,  0.0029, -0.0480,  ...,  0.0063, -0.0168,  0.0569],\n",
              "                      [ 0.0486,  0.0068,  0.0048,  ...,  0.0004, -0.0283,  0.0581],\n",
              "                      [ 0.0106, -0.0050, -0.0381,  ...,  0.0137,  0.0040,  0.0550]])),\n",
              "             ('encoder.layers.6.self_attn.linears.1.bias',\n",
              "              tensor([-0.0169,  0.0087,  0.0264,  ...,  0.0211, -0.0118, -0.0001])),\n",
              "             ('encoder.layers.6.self_attn.linears.2.weight',\n",
              "              tensor([[ 2.2013e-02,  2.5594e-02, -1.8543e-02,  ...,  4.4537e-04,\n",
              "                        1.2771e-02,  4.7090e-02],\n",
              "                      [-1.5438e-02,  5.5459e-03, -5.1401e-02,  ..., -3.6312e-02,\n",
              "                        2.8141e-02, -2.0746e-02],\n",
              "                      [-1.2342e-03,  3.7859e-02,  4.0668e-02,  ..., -5.3282e-04,\n",
              "                       -1.2893e-02,  1.0674e-02],\n",
              "                      ...,\n",
              "                      [ 2.4686e-02, -4.0530e-02, -8.7064e-06,  ...,  4.7257e-02,\n",
              "                        1.5868e-02, -2.0252e-04],\n",
              "                      [ 4.3792e-02, -3.4945e-02, -7.3483e-03,  ..., -1.6980e-02,\n",
              "                       -2.6048e-02, -4.6858e-02],\n",
              "                      [ 3.6992e-02, -9.2648e-04, -3.4188e-02,  ..., -1.1160e-02,\n",
              "                        4.4632e-02,  4.1095e-02]])),\n",
              "             ('encoder.layers.6.self_attn.linears.2.bias',\n",
              "              tensor([-0.0172,  0.0116,  0.0260,  ...,  0.0206, -0.0094,  0.0012])),\n",
              "             ('encoder.layers.6.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0433,  0.0168,  0.0196,  ...,  0.0170,  0.0531,  0.0564],\n",
              "                      [-0.0299, -0.0405, -0.0159,  ...,  0.0400, -0.0095,  0.0218],\n",
              "                      [-0.0150,  0.0074,  0.0310,  ...,  0.0185, -0.0045,  0.0455],\n",
              "                      ...,\n",
              "                      [-0.0243, -0.0459,  0.0273,  ...,  0.0135,  0.0050, -0.0511],\n",
              "                      [ 0.0487,  0.0087,  0.0198,  ...,  0.0491, -0.0293, -0.0021],\n",
              "                      [-0.0217,  0.0138, -0.0380,  ...,  0.0134,  0.0222,  0.0189]])),\n",
              "             ('encoder.layers.6.self_attn.linears.3.bias',\n",
              "              tensor([-0.0149,  0.0087,  0.0257,  ...,  0.0223, -0.0121,  0.0040])),\n",
              "             ('encoder.layers.6.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0329,  0.0404, -0.0101,  ..., -0.0039, -0.0074,  0.0453],\n",
              "                      [-0.0591, -0.0495, -0.0466,  ...,  0.0351, -0.0533, -0.0234],\n",
              "                      [-0.0187, -0.0199, -0.0599,  ..., -0.0121,  0.0566,  0.0021],\n",
              "                      ...,\n",
              "                      [ 0.0436,  0.0583,  0.0356,  ..., -0.0714, -0.0562, -0.0534],\n",
              "                      [-0.0141, -0.0337,  0.0341,  ..., -0.0204,  0.0118, -0.0324],\n",
              "                      [-0.0151,  0.0131,  0.0071,  ...,  0.0571, -0.0268,  0.0173]])),\n",
              "             ('encoder.layers.6.feed_forward.linears.0.bias',\n",
              "              tensor([ 0.0048, -0.0273,  0.0238,  ...,  0.0229, -0.0181,  0.0007])),\n",
              "             ('encoder.layers.6.sublayer.0.norm.a_2',\n",
              "              tensor([1.0011, 0.9885, 1.0049,  ..., 0.9991, 0.9984, 0.9957])),\n",
              "             ('encoder.layers.6.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0008, -0.0027, -0.0016,  ...,  0.0006,  0.0003, -0.0055])),\n",
              "             ('encoder.layers.6.sublayer.1.norm.a_2',\n",
              "              tensor([1.0006, 1.0176, 1.0142,  ..., 1.0187, 1.0090, 1.0200])),\n",
              "             ('encoder.layers.6.sublayer.1.norm.b_2',\n",
              "              tensor([ 0.0022, -0.0009,  0.0005,  ..., -0.0009, -0.0017, -0.0109])),\n",
              "             ('encoder.layers.7.self_attn.linears.0.weight',\n",
              "              tensor([[ 0.0415, -0.0502,  0.0314,  ..., -0.0246,  0.0319, -0.0419],\n",
              "                      [ 0.0004, -0.0151,  0.0027,  ...,  0.0330, -0.0508,  0.0065],\n",
              "                      [ 0.0266, -0.0052,  0.0128,  ..., -0.0530,  0.0310,  0.0229],\n",
              "                      ...,\n",
              "                      [ 0.0434, -0.0383,  0.0128,  ..., -0.0344,  0.0485,  0.0355],\n",
              "                      [ 0.0281,  0.0099, -0.0159,  ..., -0.0047, -0.0039, -0.0525],\n",
              "                      [ 0.0062,  0.0582, -0.0270,  ...,  0.0201, -0.0454,  0.0029]])),\n",
              "             ('encoder.layers.7.self_attn.linears.0.bias',\n",
              "              tensor([-0.0176,  0.0078,  0.0207,  ...,  0.0231, -0.0180,  0.0009])),\n",
              "             ('encoder.layers.7.self_attn.linears.1.weight',\n",
              "              tensor([[-0.0205,  0.0258, -0.0591,  ...,  0.0372,  0.0489, -0.0174],\n",
              "                      [-0.0140, -0.0086, -0.0219,  ..., -0.0418, -0.0116,  0.0128],\n",
              "                      [-0.0163, -0.0369, -0.0019,  ...,  0.0043, -0.0417,  0.0194],\n",
              "                      ...,\n",
              "                      [ 0.0168,  0.0027,  0.0067,  ...,  0.0372,  0.0201,  0.0168],\n",
              "                      [-0.0040,  0.0020,  0.0078,  ...,  0.0070, -0.0408,  0.0410],\n",
              "                      [ 0.0233,  0.0181,  0.0289,  ..., -0.0459, -0.0369,  0.0109]])),\n",
              "             ('encoder.layers.7.self_attn.linears.1.bias',\n",
              "              tensor([-1.7002e-02,  8.6146e-03,  2.6473e-02,  ...,  2.1091e-02,\n",
              "                      -1.1790e-02, -9.4817e-05])),\n",
              "             ('encoder.layers.7.self_attn.linears.2.weight',\n",
              "              tensor([[-0.0274, -0.0117,  0.0156,  ..., -0.0021, -0.0383,  0.0525],\n",
              "                      [-0.0238,  0.0236,  0.0402,  ..., -0.0225, -0.0038,  0.0456],\n",
              "                      [-0.0248, -0.0096, -0.0083,  ...,  0.0400, -0.0035, -0.0113],\n",
              "                      ...,\n",
              "                      [ 0.0236,  0.0268,  0.0111,  ..., -0.0040,  0.0215, -0.0047],\n",
              "                      [ 0.0521,  0.0311, -0.0321,  ...,  0.0266, -0.0462,  0.0259],\n",
              "                      [ 0.0511,  0.0074, -0.0036,  ...,  0.0196, -0.0540, -0.0315]])),\n",
              "             ('encoder.layers.7.self_attn.linears.2.bias',\n",
              "              tensor([-0.0211,  0.0076,  0.0266,  ...,  0.0197, -0.0133,  0.0024])),\n",
              "             ('encoder.layers.7.self_attn.linears.3.weight',\n",
              "              tensor([[ 0.0529, -0.0472,  0.0083,  ...,  0.0468, -0.0560,  0.0100],\n",
              "                      [-0.0357, -0.0032, -0.0371,  ..., -0.0072, -0.0360,  0.0309],\n",
              "                      [-0.0541,  0.0248,  0.0366,  ..., -0.0142,  0.0107,  0.0064],\n",
              "                      ...,\n",
              "                      [-0.0342,  0.0100,  0.0596,  ...,  0.0302,  0.0543,  0.0160],\n",
              "                      [-0.0296, -0.0073,  0.0507,  ..., -0.0433, -0.0249,  0.0418],\n",
              "                      [ 0.0404, -0.0514, -0.0350,  ..., -0.0391,  0.0358,  0.0388]])),\n",
              "             ('encoder.layers.7.self_attn.linears.3.bias',\n",
              "              tensor([-0.0212,  0.0110,  0.0190,  ...,  0.0253, -0.0120, -0.0002])),\n",
              "             ('encoder.layers.7.feed_forward.linears.0.weight',\n",
              "              tensor([[-0.0318,  0.0171,  0.0198,  ..., -0.0306,  0.0412,  0.0345],\n",
              "                      [ 0.0067, -0.0550, -0.0283,  ..., -0.0234, -0.0186,  0.0460],\n",
              "                      [-0.0111, -0.0411, -0.0221,  ..., -0.0421,  0.0293, -0.0049],\n",
              "                      ...,\n",
              "                      [-0.0244, -0.0114, -0.0253,  ..., -0.0265,  0.0360, -0.0064],\n",
              "                      [ 0.0487,  0.0088,  0.0104,  ..., -0.0606, -0.0323,  0.0488],\n",
              "                      [ 0.0047,  0.0499, -0.0618,  ...,  0.0134,  0.0307, -0.0565]])),\n",
              "             ('encoder.layers.7.feed_forward.linears.0.bias',\n",
              "              tensor([-0.0090, -0.0247,  0.0237,  ...,  0.0224, -0.0201, -0.0002])),\n",
              "             ('encoder.layers.7.sublayer.0.norm.a_2',\n",
              "              tensor([1.0016, 0.9854, 1.0058,  ..., 1.0033, 1.0080, 1.0003])),\n",
              "             ('encoder.layers.7.sublayer.0.norm.b_2',\n",
              "              tensor([ 0.0048,  0.0005,  0.0139,  ..., -0.0018,  0.0017,  0.0095])),\n",
              "             ('encoder.layers.7.sublayer.1.norm.a_2',\n",
              "              tensor([1.0213, 1.0487, 1.0298,  ..., 1.0221, 1.0218, 1.0294])),\n",
              "             ('encoder.layers.7.sublayer.1.norm.b_2',\n",
              "              tensor([-0.0016, -0.0061, -0.0017,  ...,  0.0038, -0.0018,  0.0005])),\n",
              "             ('encoder.norm.a_2',\n",
              "              tensor([1.0593, 1.0204, 1.0305,  ..., 1.0185, 1.0413, 1.0205])),\n",
              "             ('encoder.norm.b_2',\n",
              "              tensor([-0.0394,  0.0058, -0.0052,  ..., -0.0009,  0.0279,  0.0139])),\n",
              "             ('src_embed.lut.weight',\n",
              "              tensor([[-0.0212,  0.0483, -0.0148,  ..., -0.0377, -0.0621,  0.0726],\n",
              "                      [ 0.0341, -0.0600, -0.0083,  ...,  0.0175,  0.0323,  0.0712],\n",
              "                      [ 0.0744,  0.0237, -0.0668,  ...,  0.0729, -0.0232,  0.0124],\n",
              "                      ...,\n",
              "                      [ 0.0659, -0.0425,  0.0024,  ..., -0.0373, -0.0127,  0.0401],\n",
              "                      [-0.0683, -0.0583,  0.0357,  ...,  0.0517, -0.0268, -0.0253],\n",
              "                      [-0.0371, -0.0061,  0.0745,  ..., -0.0188,  0.0306, -0.0243]])),\n",
              "             ('src_embed.lut.bias',\n",
              "              tensor([-0.1589,  0.1621, -0.1299,  ...,  0.0922,  0.1594, -0.1628])),\n",
              "             ('generator.proj.weight',\n",
              "              tensor([[ 0.0686, -0.0564, -0.0706,  ..., -0.0594,  0.0197,  0.0588]])),\n",
              "             ('generator.proj.bias', tensor([0.0254]))])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Duj3iwpOlL4k"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.autograd import Variable\n",
        "import torch.optim as optim\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nysqNVuVlREw"
      },
      "source": [
        "opt = optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999))\n",
        "criterion = nn.MSELoss()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KssAGw5omzgk"
      },
      "source": [
        "batch_size = 32\n",
        "\n",
        "# Formal charges are one-hot encoded to keep compatibility with the pre-trained weights.\n",
        "# If you do not plan to use the pre-trained weights, we recommend to set one_hot_formal_charge to False.\n",
        "X, y = load_data_from_df('../data/freesolv/freesolv.csv', one_hot_formal_charge=True)\n",
        "data_loader = construct_loader(X, y, batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xA2ZEUj8lXme"
      },
      "source": [
        "def train_epoch(model, opt, criterion):\n",
        "    #model.cuda()\n",
        "    model.train()\n",
        "    losses = []\n",
        "    for batch in data_loader:\n",
        "        adjacency_matrix, node_features, distance_matrix, y = batch\n",
        "        batch_mask = torch.sum(torch.abs(node_features), dim=-1) != 0\n",
        "        opt.zero_grad()\n",
        "        output = model(node_features, batch_mask, adjacency_matrix, distance_matrix, None)\n",
        "        loss = criterion(output, y)\n",
        "        loss.backward()\n",
        "        losses.append(loss.data.numpy())\n",
        "        opt.step()        \n",
        "        losses.append(loss.data.numpy())\n",
        "    return losses\n",
        "\n",
        "\n",
        "       "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aTfZAd6emJ5J",
        "outputId": "8ff0c69a-6562-4e8b-eec8-abdd27b65336",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "e_losses = []\n",
        "num_epochs = 2\n",
        "for e in range(num_epochs):\n",
        "    e_losses += train_epoch(model, opt, criterion)\n",
        "plt.plot(e_losses)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f9debcd62e8>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcPklEQVR4nO3dbYxc133f8e9/ZnZmdpe7y11yTdEkZcoQbVl2EkllJSVyE9dqUkk2QrlQXAduTDgq+EZunTqFrbgvhLpIYQNpZBspDAiWGjpwXAuyUQmp2lSQlMoBatmk5ei5JiNX0tKkuKKWDyJ3Zufh3xf3zHJE7ezDzOy9c3l/H4DY+zQzdy93f3P2P+eeY+6OiIhkQy7pExARkfgo9EVEMkShLyKSIQp9EZEMUeiLiGRIIekTWM7mzZt9586dSZ+GiEiqHDx48HV3n15q30CH/s6dOzlw4EDSpyEikipm9nKnfSrviIhkiEJfRCRDFPoiIhmi0BcRyRCFvohIhij0RUQyRKEvIpIhA91PPwn3//hVZubOLbnPzLjtH2xnx9RIzGclItIfCv02pys1Pv+9pwEwe/t+dzi3UOfffeTKmM9MRKQ/FPpt5hcaAPzxxz7AJ69719v23/Dlxzjx5kLcpyUi0jcr1vTN7D4zO25mz7ZtmzKzR8zsUPg6GbabmX3dzA6b2dNmdk3bY/aG4w+Z2d71+XZ6U6lFoV8u5JfcPzVa5I1zCn0RSa/VfJD758BNF2y7E3jU3XcBj4Z1gJuBXeHfPuAbEL1JAHcB1wHXAne13igGSaXWBKA8tEzon1Xoi0h6rRj67v4E8MYFm/cA+8PyfuDWtu3f8sgPgY1mthX4p8Aj7v6Gu88Bj/D2N5LELbb0h5a+LAp9EUm7brtsbnH3o2H5GLAlLG8DXm07biZs67T9bcxsn5kdMLMDs7OzXZ5ed86Hvlr6InJx6rmfvrs74H04l9bz3ePuu9199/T0ksNBr5tKfeXyzrmFxuKbg4hI2nQb+q+Fsg3h6/Gw/Qiwo+247WFbp+0DpdV7Z7nyDqDWvoikVreh/xDQ6oGzF3iwbfunQi+e64FToQz018Bvmdlk+AD3t8K2gVKtL1/emRxR6ItIuq3YT9/MvgN8CNhsZjNEvXC+DNxvZrcDLwMfD4c/DNwCHAbOAZ8GcPc3zOw/AD8Ox33J3S/8cDhxK9X0N21Q6ItIuq0Y+u7+ux123bjEsQ7c0eF57gPuW9PZxWyxy2Zh6T+AWi39OfXVF5GU0oBrbVZs6Yeavu7KFZG0Uui3WenmrInhIXKmlr6IpJdCv818rUExnyOfW2K0NSCXMyZHipxQTV9EUkqh36ZSa1Dq0F2zZXK0yJxCX0RSSqHfplpvdCzttEyNqqUvIuml0G9TqTU73pjVskktfRFJMYV+m0qt0XFY5ZZJjb8jIimm0G9TqTUYLi4f+ptGi8ydW6DZ7NtwQyIisVHot5lfTUt/pEjT4dR8LaazEhHpH4V+m0qtuWLvndZQDPowV0TSSKHfplJbufeOhmIQkTRT6Lep1pur6rIJGopBRNJJod8m6r2z/CVphb5a+iKSRgr9NqvpvaOJVEQkzRT6beZXUdMvD+UZKeYV+iKSSgr9wN2jO3JXKO+AJkgXkfRS6AfVMCl6aYWWPij0RSS9FPpBdYWx9Nsp9EUkrRT6QWVxUvRVlHdGFPoikk4K/aA1VeKwWvoichFT6AfzK8yP225ytMh8rcH8QmO9T0tEpK8U+sH5+XFXviStCdLf0A1aIpIyCv2gVd5ZaZRNiFr6gCZTEZHUUegHrdBfTZfNVktfI22KSNoo9IO1lHem1NIXkZRS6AfV+tp674Ba+iKSPgr9oNUTZzW9d8bLQ+Rzppa+iKSOQj+orKHLZi5nTI4MqaUvIqmj0A8q9dXX9CEq8ailLyJp01Pom9m/MbPnzOxZM/uOmZXN7DIze9LMDpvZd82sGI4thfXDYf/OfnwD/bKWLpsQTZuou3JFJG26Dn0z2wb8a2C3u38AyAOfAL4C3O3ulwNzwO3hIbcDc2H73eG4gVGpNSnmc+RytqrjN20o6uYsEUmdXss7BWDYzArACHAU+DDwQNi/H7g1LO8J64T9N5rZ6hI2BpVag9IqSzuglr6IpFPXoe/uR4A/AV4hCvtTwEHgpLvXw2EzwLawvA14NTy2Ho7f1O3r91ul1lhVd82WTaNFTp5boNH0dTwrEZH+6qW8M0nUer8MeCcwCtzU6wmZ2T4zO2BmB2ZnZ3t9ulWrrGKqxHaTo0WaDqfma+t4ViIi/dVLeeefAD9391l3rwHfB24ANoZyD8B24EhYPgLsAAj7J4ATFz6pu9/j7rvdfff09HQPp7c2lVpz1T13QBOki0g69RL6rwDXm9lIqM3fCDwPPA7cFo7ZCzwYlh8K64T9j7n7wNRGKvW1tfQV+iKSRr3U9J8k+kD2J8Az4bnuAb4AfM7MDhPV7O8ND7kX2BS2fw64s4fz7rtKrbHq7pqg0BeRdCqsfEhn7n4XcNcFm18Crl3i2ArwO7283nqq1JqMlVd/ORT6IpJGuiM3WGvvncmRMNKm+uqLSIoo9IO19t4pD+UZLeY58aZCX0TSQ6EfrLX3DsDUhqJa+iKSKgr9YK29dwCmRooaaVNEUkWhH6y1vAMaaVNE0kehD7h7VN4prO1yTI5q/B0RSReFPlBtjaVfXFtLf5NCX0RSRqHP2sfSb5kaLTFfayxOtSgiMugU+kQ9d2B1UyW2mxodAuDE2Wrfz0lEZD0o9GmfH3eNXTZHSwDMndVImyKSDgp9ou6aoJa+iFz8FPq0l3e6bOnrBi0RSQmFPix+ENvNzVmAhmIQkdRQ6NN9eWd8uEA+Z2rpi0hqKPSBapddNs1ME6SLSKoo9Om+pg+6QUtE0kWhT3uXzbW19AEmR4cU+iKSGgp9egv9TaMlhb6IpIZCH5gP5Z21zJzVopa+iKSJQp/zLf3SGkfZhKiv/sn5Go2m9/u0RET6TqFP1GWzWMiRy9maHzs1MoQ7nFS3TRFJAYU+UO1iLP2WqQ26K1dE0kOhT3ezZrXorlwRSROFPj2G/mgU+mrpi0gaKPSB+Vqjq547cD70NUG6iKSBQp/ojtxu7saFqMsmoAnSRSQVFPpE5Z1Sly39UiHPhlJBLX0RSQWFPlCpN7uu6UNU4lFLX0TSQKFPNMpmt102IQp9tfRFJA0U+vTWewdCS1+9d0QkBXoKfTPbaGYPmNmLZvaCmf2qmU2Z2SNmdih8nQzHmpl93cwOm9nTZnZNf76F3vXSewei0H9D/fRFJAV6bel/Dfif7n4F8CvAC8CdwKPuvgt4NKwD3AzsCv/2Ad/o8bX7ppfeOxBCXy19EUmBrpPOzCaAXwfuBXD3BXc/CewB9ofD9gO3huU9wLc88kNgo5lt7frM+6gf5Z1Krcm5hXofz0pEpP96aelfBswC/8XMnjKzb5rZKLDF3Y+GY44BW8LyNuDVtsfPhG1vYWb7zOyAmR2YnZ3t4fRWx92p1ptdd9mE80MxaIhlERl0vYR+AbgG+Ia7Xw2c5XwpBwB3d2BNYw67+z3uvtvdd09PT/dweqtTrXc/VWJL665chb6IDLpeQn8GmHH3J8P6A0RvAq+1yjbh6/Gw/wiwo+3x28O2RFW6nBS93aRCX0RSouvQd/djwKtm9t6w6UbgeeAhYG/Ythd4MCw/BHwq9OK5HjjVVgZKzHwI/eFi96G/SaEvIilR6PHx/wr4tpkVgZeATxO9kdxvZrcDLwMfD8c+DNwCHAbOhWMTV6n1Xt5RS19E0qKn0Hf3nwK7l9h14xLHOnBHL6+3HvpR3hkvFyjkjBePneGpV+aWPOY9W8YYLfX6Hisi0pvMp9Bi6PfQe8fM2LqxzAMHZ3jg4MySx3zs6m3c/c+v6vo1RET6QaEfyjulHso7AH/x+9fx8xNnl9z3lf/xIr84Od/T84uI9INCv957Sx9g5+ZRdm4eXXLft3/4CjNz53p6fhGRfsj8gGuVhdB7p8fQX874cIHT87V1e34RkdVS6Peppb+cieEhTlc0RIOIJE+h34cumyuZGB7izWqdeqO5bq8hIrIaCv0+dNlcycRwNI+uWvsikjSF/mJLf/1D/5Tq+iKSMIV+aOmXepgucSXjZYW+iAwGhX69QamQI5ezdXuNiZFQ3lHoi0jCFPoLvU2gshoq74jIoFDo9zhV4moo9EVkUCj062rpi0h2KPRrjXXtrglRz6BiIaeavogkTqEfQ3kHoh48aumLSNIU+rX1L+8ATAwXOF1R6ItIshT6sYW+WvoikjyFfkzlHYW+iAwChX4MvXdAoS8ig0GhH0PvHQihf06hLyLJUujH1XtneIgz1TrNpq/7a4mIdKLQrzUoF+Np6bvDmaqGVxaR5GQ69JtNp1pvxlLeGR/WoGsikrxMh361vv5j6bdoKAYRGQSZDv3FWbNi6rIJCn0RSVa2Qz+GSdFbFPoiMgiyHfoxTIreopq+iAyCjId+1NIfVktfRDIi06E/35ofN4bQHy3myedMoS8iicp06C9+kBtDl00z01AMIpK4nkPfzPJm9pSZ/VVYv8zMnjSzw2b2XTMrhu2lsH447N/Z62v3qhpjTR80/o6IJK8fafdZ4IW29a8Ad7v75cAccHvYfjswF7bfHY5L1Pkum+vf0gcYLxc4XdEduSKSnJ5C38y2Ax8BvhnWDfgw8EA4ZD9wa1jeE9YJ+28Mxycmzi6bEPXgUUtfRJLUa0v/q8DngWZY3wScdPdWc3YG2BaWtwGvAoT9p8Lxb2Fm+8zsgJkdmJ2d7fH0ltfqshlH7x2IyjvqsikiSeo69M3so8Bxdz/Yx/PB3e9x993uvnt6erqfT/028wvx3ZELqumLSPIKPTz2BuC3zewWoAyMA18DNppZIbTmtwNHwvFHgB3AjJkVgAngRA+v37O4yzut0Hd3Eq5siUhGdd3Edfc/cvft7r4T+ATwmLt/EngcuC0cthd4MCw/FNYJ+x9z90QHl2+Vd0qF+Fr6jaZzNvyFISISt/VIuy8AnzOzw0Q1+3vD9nuBTWH754A71+G116Raa1Aq5GJrdWsoBhFJWi/lnUXu/jfA34Tll4BrlzimAvxOP16vXyq1eObHbWkfiuGdG4dje10RkZaM35HbjK3nDmj8HRFJXqZDf77WiK3nDij0RSR5mQ79JMs7IiJJyHbo15uxjLDZog9yRSRp2Q79WoNyTN01AcZKBcwU+iKSnEyHfjXm8k4uZ4yVCirviEhiMh36lVoz1g9yASZGNBSDiCQn06E/X2vE2mUTNP6OiCQr06Efd+8dUOiLSLIU+gmEviZSEZGkZDv0601KMdf0x8tq6YtIcjIb+s2ms1BvxjIpejuVd0QkSZkN/Wq9NSl6vKE/PjzEQr25OD+viEicMhv68yF0h+PusqmhGEQkQZkN/VZLO4kPckF35YpIMhT6CZR3QC19EUlGhkO/VdNXeUdEsiO7oR8mRY9zlE1Q6ItIsrIb+q3yTgJdNkGhLyLJyHzoDxdjrumXo2mJFfoikoQMh34yNf1CPseGUoHT8xqKQUTil+HQT6a8A1FrXy19EUlChkM/mTtyIeq2qdAXkSRkOPRb/fTjvwQTw0O6OUtEEpHd0K8nc3MWaNA1EUlOdkN/IfTTj3Fi9JZoTH2FvojEL7uhX4/mxzWz2F9bLX0RSUp2Qz+BWbNaxoeHOLfQoNZoJvL6IpJd2Q79BLprgu7KFZHkdB36ZrbDzB43s+fN7Dkz+2zYPmVmj5jZofB1Mmw3M/u6mR02s6fN7Jp+fRPdqNSaifTcAYW+iCSnl9SrA3/o7lcC1wN3mNmVwJ3Ao+6+C3g0rAPcDOwK//YB3+jhtXuWZHlHoS8iSek69N39qLv/JCyfAV4AtgF7gP3hsP3ArWF5D/Atj/wQ2GhmW7s+8x7NJ1zTB02kIiLx60t9w8x2AlcDTwJb3P1o2HUM2BKWtwGvtj1sJmy78Ln2mdkBMzswOzvbj9NbUlXlHRHJoJ5Tz8w2AN8D/sDdT7fvc3cHfC3P5+73uPtud989PT3d6+l1VKkn2dKPRtpUS19E4tZT6JvZEFHgf9vdvx82v9Yq24Svx8P2I8COtodvD9sSod47IpJFvfTeMeBe4AV3/9O2XQ8Be8PyXuDBtu2fCr14rgdOtZWBYpdk751SIU95KKfQF5HYFXp47A3A7wHPmNlPw7YvAl8G7jez24GXgY+HfQ8DtwCHgXPAp3t47Z4l2XsHWoOuaUx9EYlX16Hv7n8LdBrD4MYljnfgjm5fr9+S7L0DGopBRJKR2Ttyo947yYX+eFmhLyLxy2ToN5rOQiO5mj6opS8iychk6FcTHEu/RaEvIknIZOgvTpWYwFj6LeOaPUtEEpDR0B+Mlv6Zap1Gc033romI9CSToT8fQn+4mGzoA5zRDFoiEqNMhn6rpV9K6I5cOD/omur6IhKnjIZ+qOkn3HsHFPoiEq9Mhn51QGr6oNAXkXhlMvQrA9JlE9BQDCISq2yGvso7IpJRmQz9+YXQe2cAWvoKfRGJUyZDfxDKO+WhHEN5U+iLSKyyGfqLd+QmF/pmpqEYRCR2GQ390E8/wZo+aCgGEYlfJkO/WmtgBqUEx96BMJGK7sgVkRhlMvQr9SalQo5oxsfkqLwjInHLZOjPLzQS7bnTotAXkbhlMvSTnh+3RbNniUjcshn69WSnSmyZCB/kNjW8sojEJJuhX2sk/iEuRKHfdDi7oKEYRCQeySdfAgalvKO7ckUkbpkM/Wot2UnRWzSmvojELfnkS8B8bXB674BCX0Tik8nQH5TyzvhwAUB35YpIbLIZ+vXBCH2NqS8icctm6A9ITV/lHRGJWyHpE0hC1GUz+Zb+hlKBfC7e4ZUbTefoqXm8w60B+ZyxeUOJ4gB0aRWR/stk6Ee9d5IPfTNjvFxY99B/7XSFJ342yxOHXudvD80yd27l19u8ocTWiTKXTJR550SZSyaGecdYiUJ+6fGK8jnjqh0b2T450u/TF5E+ylzoN5rOQqM5EL13IOq2udbQrzeanDi7wJllRug8dqrKDw7N8r9/NsuLx84AUZD/4yvewe53TXVsydcaTV47XeHYqQpHT1V45cQ5nnzpBKcrq/vc4d2bR/lHuzbz6++Z5vp3b2K0lLkfMZGBFvtvpJndBHwNyAPfdPcvx/n6rbH0B6GmD1Fd/9lfnOLPHju05P6zCw1mz1Q5fqbK7Jkqs2cqnDi70LE8024ob+x+1xRfuOkKfuM907xv61jXI4uerdaZPVOl2eGFzy00+NHP3+AHh2a5/8AM+//PywzljWsuneSDl29mcrS45OPM4NKpEX5p2wQbR5Y+ZlA1m87M3DwvHDvNzNx8x+M8XLOmO+7ggDsdr2X74xYaTrXeoFprstBoUq01qdYb1BrNjj8DZjA1WuSS8WEumShxycQwl4yXuWS8zPhwIfHRZSVZsYa+meWB/wz8JjAD/NjMHnL35+M6h/OhPxgt/fe/c4Lv/OgV/uR//WzJ/YWcMT1W4h1jJbZtLHPVjo2L6+PDQ3T69R0rF/iHO6f61tIeLRVWfK4PbJvg9z94GdV6g4P/b44nDr3OEz+b5T89svT3dqFW+P/S9gl+edsE7982sfhhdz+5O/WmU2s0qdWjv/zqzWi5sUwQz51b4IWjp3nx6Jno67EzvFld/55XpUKOYiFHqZCnVMhRKuQYyufolN3ucPDlOV5/c+Ft+8pDuWV/9keLBabHSmzeUGJ6rMjmDaXFfyPFzo8rFnJsKBUYKxcYKw8xVi7ENnx5td7gTKXOm5U6Zyr1ZYc1aTad+VqDcwsN5msN5hfC8kKdSr25+AZ9IXeo1qM33Gq9SaX21q+dfmw8PNiJ3uSbzdabvq/YcPuN907zxVvet5pLsCZxt/SvBQ67+0sAZvZfgT1AX0P/xWOn+cxfPrXkvnojTJU4IC39//ixD/ClPe/vuD9vRi6XrpZZqZDn1y7fzK9dvpk7b76C05Xa4pvthRpN5++Pn+XpIyd59sgp/m7mJP/9maOL+wsdvvdehqhr9DjA3VipwBVbx/hn12zjikvGed/WMXZuGl32/yln0Wc4RtQSz4UwXCkTi/nug7Nab3D8dJXXTkelulbZrhZ+By7kRPeMvP7mAjNz5/jpq3Or/qtyKUN5Y6w8FIX/MseZGblcdE2M8LXtei2l3nTOVGqcrtRZqC/9/axFa1Kl3DLXuhTeeMtDb/26obT8X0/R9/T278sMOn+H8I6xUg/fUWdxh/424NW29RnguvYDzGwfsA/g0ksv7epFyoU8790y1nH/1ZdO8sFd0109d7+ZGUMdPhy9WIyXhxgvd26xb50Y5oO7Ni+uv3F2gWeOnOK5X5zi7DIt6eV+YZaTs6hlOpTPUcjnKOaNoXy0nl8muEdLBa64ZIztk8OpKJGUCnl2TI2wY6r7D9cbTeeNswu8/maV6jLhWq2F1na1vhjGreVqrfPjWqUud49awqH8FZXCOr/b5Cx6Qxkvv/Wvi7HyEKPFPJ1+NAxjuJhnpJhneCj6OlIsUB5KflKluNhyF7bvL2Z2G3CTu//LsP57wHXu/pmljt+9e7cfOHAgtvMTEbkYmNlBd9+91L64axxHgB1t69vDNhERiUHcof9jYJeZXWZmReATwEMxn4OISGbFWtN397qZfQb4a6Ium/e5+3NxnoOISJbF3k/f3R8GHo77dUVEJKMDromIZJVCX0QkQxT6IiIZotAXEcmQWG/OWiszmwVe7uEpNgOv9+l0Lia6Lp3p2nSma9PZoF2bd7n7ksMODHTo98rMDnS6Ky3LdF0607XpTNemszRdG5V3REQyRKEvIpIhF3vo35P0CQwoXZfOdG0607XpLDXX5qKu6YuIyFtd7C19ERFpo9AXEcmQizL0zewmM/u/ZnbYzO5M+nySZGb3mdlxM3u2bduUmT1iZofC18kkzzEpZrbDzB43s+fN7Dkz+2zYnunrY2ZlM/uRmf1duC7/Pmy/zMyeDL9X3w3Do2eSmeXN7Ckz+6uwnpprc9GFftvk6zcDVwK/a2ZXJntWifpz4KYLtt0JPOruu4BHw3oW1YE/dPcrgeuBO8LPStavTxX4sLv/CnAVcJOZXQ98Bbjb3S8H5oDbEzzHpH0WeKFtPTXX5qILfdomX3f3BaA1+XomufsTwBsXbN4D7A/L+4FbYz2pAeHuR939J2H5DNEv8TYyfn088mZYHQr/HPgw8EDYnrnr0mJm24GPAN8M60aKrs3FGPpLTb6+LaFzGVRb3P1oWD4GbEnyZAaBme0ErgaeRNenVb74KXAceAT4e+Cku7dmqs/y79VXgc8DrRnfN5Gia3Mxhr6sgUd9djPdb9fMNgDfA/7A3U+378vq9XH3hrtfRTSP9bXAFQmf0kAws48Cx939YNLn0q3YZ86KgSZfX9lrZrbV3Y+a2Vai1lwmmdkQUeB/292/Hzbr+gTuftLMHgd+FdhoZoXQos3q79UNwG+b2S1AGRgHvkaKrs3F2NLX5OsrewjYG5b3Ag8meC6JCbXYe4EX3P1P23Zl+vqY2bSZbQzLw8BvEn3e8ThwWzgsc9cFwN3/yN23u/tOomx5zN0/SYquzUV5R254F/4q5ydf/+OETykxZvYd4ENEQ7++BtwF/DfgfuBSoqGrP+7uF37Ye9Ezsw8CPwCe4Xx99otEdf3MXh8z+2WiDyPzRA3D+939S2b2bqKOEVPAU8C/cPdqcmeaLDP7EPBv3f2jabo2F2Xoi4jI0i7G8o6IiHSg0BcRyRCFvohIhij0RUQyRKEvIpIhCn0RkQxR6IuIZMj/Bwqkpfj3PPKnAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CDFdH11Un59e",
        "outputId": "0da27465-5434-4c79-f5da-f486e70af777",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "!nvcc --version"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2019 NVIDIA Corporation\n",
            "Built on Sun_Jul_28_19:07:16_PDT_2019\n",
            "Cuda compilation tools, release 10.1, V10.1.243\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fzkGxAKNo_84",
        "outputId": "63a6382e-e8d5-43e7-df35-20f8931f760f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 360
        }
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon Aug 24 08:36:35 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 450.57       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   46C    P0    57W / 149W |    389MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7gJb6YE4R1GJ"
      },
      "source": [
        "model.cuda()\n",
        "for batch in data_loader:\n",
        "    adjacency_matrix, node_features, distance_matrix, y = batch\n",
        "    batch_mask = torch.sum(torch.abs(node_features), dim=-1) != 0\n",
        "    output = model(node_features, batch_mask, adjacency_matrix, distance_matrix, None)\n",
        "    out.append(output)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WZMod10BlLZ-"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IcVisAQQZMZR"
      },
      "source": [
        "for batch in data_loader:\n",
        "\n",
        "    adjacency_matrix, node_features, distance_matrix, y = batch\n",
        "    batch_mask = torch.sum(torch.abs(node_features), dim=-1) != 0\n",
        "    output = model1(node_features, batch_mask, adjacency_matrix, distance_matrix, None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SxvLG-pI8Xny"
      },
      "source": [
        "\n",
        "for batch in data_loader:\n",
        "\n",
        "    adjacency_matrix, node_features, distance_matrix, y = batch\n",
        "    batch_mask = torch.sum(torch.abs(node_features), dim=-1) != 0\n",
        "    output = model(node_features, batch_mask, adjacency_matrix, distance_matrix, None)\n",
        "\n",
        "\n",
        "    \n",
        "x=output\n",
        "mask=batch_mask\n",
        "mask = mask.unsqueeze(-1).float()\n",
        "out_masked = x * mask\n",
        "out_sum = x.sum(dim=1)\n",
        "mask_sum = mask.sum(dim=(1))\n",
        "out_avg_pooling = out_sum / mask_sum\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c-DL45h2tfVw"
      },
      "source": [
        "x=output\n",
        "mask=batch_mask\n",
        "mask = mask.unsqueeze(-1).float()\n",
        "out_masked = x * mask\n",
        "out_sum = x.sum(dim=1)\n",
        "mask_sum = mask.sum(dim=(1))\n",
        "out_avg_pooling = out_sum / mask_sum\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BfHTStWICoud",
        "outputId": "27d1656d-0b3c-4703-c4be-2f191433f789",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "batch_mask.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 10])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sb7spNZaC811",
        "outputId": "8966cb24-fb7c-4d6b-a0c2-2cb26da0cc23",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 10, 1024])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_qITFFA8Zpl",
        "outputId": "d31c2a25-277f-4858-954f-a2d79cd7563a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "output.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 10, 1024])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9aimloOWDeCJ",
        "outputId": "e92ee566-d4a8-4105-a442-107c77256bd9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "out_sum.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I-h3pLYiDkrY",
        "outputId": "0adb4078-d295-4092-850d-ddc723405d54",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "mask_sum.shape\n",
        "mask_sum"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([24, 21, 27, 30, 22])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nLPTRYyQr3t3",
        "outputId": "fda1853c-713b-4239-ed5d-4ba8e5baa267",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(batch)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 120
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g_I7IDmekzpD",
        "outputId": "10a1fd76-b966-4ae6-9d99-3ad4380d3028",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[-1.3529, -0.3415,  1.4923,  ..., -1.0154, -0.6525,  2.0210],\n",
              "         [-0.6542, -0.8702,  0.1097,  ..., -0.9197, -0.7331,  2.4551],\n",
              "         [-0.5797, -1.1829, -0.0424,  ..., -0.8563, -0.4092,  2.2854],\n",
              "         ...,\n",
              "         [-0.2007, -0.8522, -0.3655,  ..., -0.4852, -0.1806,  1.4063],\n",
              "         [-0.2007, -0.8522, -0.3655,  ..., -0.4852, -0.1806,  1.4063],\n",
              "         [-0.2007, -0.8522, -0.3655,  ..., -0.4852, -0.1806,  1.4063]],\n",
              "\n",
              "        [[-0.8367,  0.1325,  1.3578,  ..., -0.6594, -0.8688,  1.5665],\n",
              "         [-0.2754, -1.1232, -0.0911,  ..., -0.9485, -1.5469,  2.0030],\n",
              "         [-0.3617, -1.2107, -0.0183,  ..., -0.8000, -1.4827,  2.1218],\n",
              "         ...,\n",
              "         [ 0.0589, -0.4579,  0.0137,  ..., -0.7118, -1.5851,  1.5854],\n",
              "         [ 0.0589, -0.4579,  0.0137,  ..., -0.7118, -1.5851,  1.5854],\n",
              "         [ 0.0589, -0.4579,  0.0137,  ..., -0.7118, -1.5851,  1.5854]]],\n",
              "       grad_fn=<AddBackward0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aC5oea8DDwRV"
      },
      "source": [
        "out_sum = out_masked.sum(dim=1)\n",
        "mask_sum = mask.sum(dim=(1))\n",
        "out_avg_pooling = out_sum / mask_sum"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y0ZIAOX0mhLy",
        "outputId": "f85d1bda-4f71-4472-d2cc-0bb29b8776aa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "output.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 10, 1024])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SVj6zGTKmmFH",
        "outputId": "cff3b02a-3e9e-415c-da58-a35cfe2f4e38",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "x.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10, 1024])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zUCWYqvfm0Ol",
        "outputId": "b4567856-bc12-4a88-c4e8-b1b99fc11618",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "mask=output[1]\n",
        "mask.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10, 1024])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7kdUFdUcn6qh"
      },
      "source": [
        "x=output[0]\n",
        "mask=output[1]\n",
        "out_masked=x*mask"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRHLth3eoRqY"
      },
      "source": [
        "mask=mask.unsqueeze(-1).float()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1M6lz-5NoYjv",
        "outputId": "b687bc5f-6a2a-485e-819f-7b47e5bf524d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "mask.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10, 1024, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ulq4hsypodgG"
      },
      "source": [
        "x=mask.unsqueeze(-1).float()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nNbm_J3BCgmD"
      },
      "source": [
        "x=output\n",
        "mask=batch_mask\n",
        "#x=x.unsqueeze(-1).float()\n",
        "mask = mask.unsqueeze(-1).float()\n",
        "out_masked = x * mask\n",
        "out_sum = x.sum(dim=1)\n",
        "mask_sum = mask.sum(dim=(1))\n",
        "out_avg_pooling = out_sum / mask_sum\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7PlLXPl-puWa",
        "outputId": "7208e357-4b3c-446b-c26b-36d623b07067",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "out_avg_pooling.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 1024])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 129
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oBmnWpfTpnIc",
        "outputId": "21594efc-9fe5-4f5e-aa91-1886cd0fecf0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "out_masked.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 10, 1024])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 130
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZcIRiwW9pYzS",
        "outputId": "79f55e10-59b0-40a8-ced2-c3f1a1a33184",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "out_avg_pooling.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10, 10])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dg4TccFvo1fE",
        "outputId": "aa423a08-b086-40c1-c31f-0aad52e0588a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "x=output[0]\n",
        "x=x.unsqueeze(-1).float()\n",
        "x.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10, 1024, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s4sC9WUDmoli"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ULMGSZi1DIoJ",
        "outputId": "1018de3d-1cac-48f1-f7d3-7853b06e6791",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 274
        }
      },
      "source": [
        "output[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "          True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "          True,  True,  True,  True, False, False, False, False, False, False],\n",
              "        [ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "          True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "          True, False, False, False, False, False, False, False, False, False],\n",
              "        [ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "          True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "          True,  True,  True,  True,  True,  True,  True, False, False, False],\n",
              "        [ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "          True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "          True,  True,  True,  True,  True,  True,  True,  True,  True,  True],\n",
              "        [ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "          True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "          True,  True, False, False, False, False, False, False, False, False]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "krMwaP9h8Dl5",
        "outputId": "315feab2-f31d-475d-b415-9b48c86ea60a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 980
        }
      },
      "source": [
        "output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[[-1.2634,  0.2185,  1.2897,  ..., -1.0203, -0.3655,  1.5253],\n",
              "          [-0.2849, -1.0072, -0.4562,  ..., -0.7751, -0.8033,  1.0816],\n",
              "          [-0.3914, -0.4924, -0.2626,  ..., -0.7474, -1.0696,  0.7292],\n",
              "          ...,\n",
              "          [ 0.2695, -1.5059, -0.9538,  ..., -0.4632, -0.4351,  0.4090],\n",
              "          [ 0.2695, -1.5059, -0.9538,  ..., -0.4632, -0.4351,  0.4090],\n",
              "          [ 0.2695, -1.5059, -0.9538,  ..., -0.4632, -0.4351,  0.4090]],\n",
              " \n",
              "         [[-1.2035,  0.2484,  1.5921,  ..., -0.7152, -0.6572,  1.2235],\n",
              "          [ 0.2839, -0.6649, -0.6582,  ..., -0.9721, -0.7759,  0.7158],\n",
              "          [-0.1604, -0.6607, -0.4911,  ..., -1.0997, -0.6646,  2.1016],\n",
              "          ...,\n",
              "          [ 0.2699, -1.5854, -1.0415,  ..., -0.8561, -0.5074,  0.4132],\n",
              "          [ 0.2699, -1.5854, -1.0415,  ..., -0.8561, -0.5074,  0.4132],\n",
              "          [ 0.2699, -1.5854, -1.0415,  ..., -0.8561, -0.5074,  0.4132]],\n",
              " \n",
              "         [[-1.4659,  0.1606,  1.5822,  ..., -0.9201, -0.8704,  0.6357],\n",
              "          [-0.6658, -1.4810, -0.6632,  ..., -1.0895, -0.6913,  0.9571],\n",
              "          [-0.5575, -1.2421, -0.7176,  ..., -0.9715, -0.7547,  0.9274],\n",
              "          ...,\n",
              "          [-0.0963, -1.1690, -1.0205,  ..., -0.5260, -0.4995, -0.0143],\n",
              "          [-0.0963, -1.1690, -1.0205,  ..., -0.5260, -0.4995, -0.0143],\n",
              "          [-0.0963, -1.1690, -1.0205,  ..., -0.5260, -0.4995, -0.0143]],\n",
              " \n",
              "         [[-1.4566,  0.2267,  1.4918,  ..., -1.0186, -0.6714,  0.9189],\n",
              "          [-0.4998, -0.7485, -0.5429,  ..., -0.9200, -0.8817,  1.5911],\n",
              "          [-0.6817, -1.1092, -0.7817,  ..., -0.9631, -0.9577,  1.1207],\n",
              "          ...,\n",
              "          [-0.8931, -0.7090, -0.2179,  ..., -0.5561, -0.7605,  0.8839],\n",
              "          [-0.9227, -0.1843,  0.0859,  ..., -0.4334, -1.0349,  0.5226],\n",
              "          [-0.8876, -0.3455, -0.0399,  ..., -0.5798, -1.0742,  0.4357]],\n",
              " \n",
              "         [[-1.2825,  0.3560,  1.4529,  ..., -1.1711, -0.6915,  0.9500],\n",
              "          [ 0.2431, -0.7375, -0.3152,  ..., -0.6891, -0.3871,  1.8180],\n",
              "          [ 0.2902, -0.7131, -0.4206,  ..., -0.7206, -0.4191,  1.7993],\n",
              "          ...,\n",
              "          [ 0.3727, -1.2503, -1.0370,  ..., -0.4891, -0.3689,  0.3982],\n",
              "          [ 0.3727, -1.2503, -1.0370,  ..., -0.4891, -0.3689,  0.3982],\n",
              "          [ 0.3727, -1.2503, -1.0370,  ..., -0.4891, -0.3689,  0.3982]]],\n",
              "        grad_fn=<AddBackward0>),\n",
              " tensor([[ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "           True,  True,  True,  True, False, False, False, False, False, False],\n",
              "         [ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "           True, False, False, False, False, False, False, False, False, False],\n",
              "         [ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "           True,  True,  True,  True,  True,  True,  True, False, False, False],\n",
              "         [ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True],\n",
              "         [ True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "           True,  True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
              "           True,  True, False, False, False, False, False, False, False, False]]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T3CEGL1cGwFF",
        "outputId": "eee3b777-b552-40c4-8dbb-559c4cf70d6f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "batch[1][0].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10, 28])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-40cFnpaHXw5",
        "outputId": "9e718bb3-baaf-47f2-b8d6-b7fdb9ea9917",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "batch_mask.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 10])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cMb7QvRK3bIc"
      },
      "source": [
        "\n",
        "adjacency_matrix, node_features, distance_matrix, y = batch\n",
        "batch_mask = torch.sum(torch.abs(node_features), dim=-1) != 0\n",
        "output = model1(node_features, batch_mask, adjacency_matrix, distance_matrix, None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BcEjb6ejvR1e"
      },
      "source": [
        "for datal"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aH7W80X7319E"
      },
      "source": [
        "adjacency_matrix, node_features, distance_matrix, y = batch\n",
        "batch_mask = torch.sum(torch.abs(node_features), dim=-1) != 0\n",
        "output = model(node_features, batch_mask, adjacency_matrix, distance_matrix, None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ITKz9DqMvgUy",
        "outputId": "cb25a855-0264-458e-bca2-53774ca1cbf0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[-1.3529, -0.3415,  1.4923,  ..., -1.0154, -0.6525,  2.0210],\n",
              "         [-0.6542, -0.8702,  0.1097,  ..., -0.9197, -0.7331,  2.4551],\n",
              "         [-0.5797, -1.1829, -0.0424,  ..., -0.8563, -0.4092,  2.2854],\n",
              "         ...,\n",
              "         [-0.2007, -0.8522, -0.3655,  ..., -0.4852, -0.1806,  1.4063],\n",
              "         [-0.2007, -0.8522, -0.3655,  ..., -0.4852, -0.1806,  1.4063],\n",
              "         [-0.2007, -0.8522, -0.3655,  ..., -0.4852, -0.1806,  1.4063]],\n",
              "\n",
              "        [[-0.8367,  0.1325,  1.3578,  ..., -0.6594, -0.8688,  1.5665],\n",
              "         [-0.2754, -1.1232, -0.0911,  ..., -0.9485, -1.5469,  2.0030],\n",
              "         [-0.3617, -1.2107, -0.0183,  ..., -0.8000, -1.4827,  2.1218],\n",
              "         ...,\n",
              "         [ 0.0589, -0.4579,  0.0137,  ..., -0.7118, -1.5851,  1.5854],\n",
              "         [ 0.0589, -0.4579,  0.0137,  ..., -0.7118, -1.5851,  1.5854],\n",
              "         [ 0.0589, -0.4579,  0.0137,  ..., -0.7118, -1.5851,  1.5854]]],\n",
              "       grad_fn=<AddBackward0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aCpNTVQ4vhtD",
        "outputId": "7eee6513-0a3b-476f-babb-71c972dcffa7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "y"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-0.0642],\n",
              "        [-0.1605]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lR15kYOdjnJ7"
      },
      "source": [
        "for batch in data_loader:\n",
        "     adjacency_matrix, node_features, distance_matrix, y = batch\n",
        "  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z0gAGtIehGhT",
        "outputId": "0830468f-5d5f-43a9-9cb9-365c9497e083",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "batch[3]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-0.0642],\n",
              "        [-0.1605]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9bU5waClg97a",
        "outputId": "ad1e752c-27e3-4649-d469-f9c0e9966652",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[-1.3529, -0.3415,  1.4923,  ..., -1.0154, -0.6525,  2.0210],\n",
              "         [-0.6542, -0.8702,  0.1097,  ..., -0.9197, -0.7331,  2.4551],\n",
              "         [-0.5797, -1.1829, -0.0424,  ..., -0.8563, -0.4092,  2.2854],\n",
              "         ...,\n",
              "         [-0.2007, -0.8522, -0.3655,  ..., -0.4852, -0.1806,  1.4063],\n",
              "         [-0.2007, -0.8522, -0.3655,  ..., -0.4852, -0.1806,  1.4063],\n",
              "         [-0.2007, -0.8522, -0.3655,  ..., -0.4852, -0.1806,  1.4063]],\n",
              "\n",
              "        [[-0.8367,  0.1325,  1.3578,  ..., -0.6594, -0.8688,  1.5665],\n",
              "         [-0.2754, -1.1232, -0.0911,  ..., -0.9485, -1.5469,  2.0030],\n",
              "         [-0.3617, -1.2107, -0.0183,  ..., -0.8000, -1.4827,  2.1218],\n",
              "         ...,\n",
              "         [ 0.0589, -0.4579,  0.0137,  ..., -0.7118, -1.5851,  1.5854],\n",
              "         [ 0.0589, -0.4579,  0.0137,  ..., -0.7118, -1.5851,  1.5854],\n",
              "         [ 0.0589, -0.4579,  0.0137,  ..., -0.7118, -1.5851,  1.5854]]],\n",
              "       grad_fn=<AddBackward0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qTbS-hHT3e6k",
        "outputId": "bd060e3a-18b6-4697-f3c9-690a6d0d2e00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(output[0][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1024"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NHTE8Dx7kSQw",
        "outputId": "7b2cde9a-d4d2-4001-ea0c-ec3101342c5f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(output[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MO950Od0O2sa"
      },
      "source": [
        "mask=batch_mask\n",
        "mask=output[1]\n",
        "x=output[0]\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CcIo9by_ka4C",
        "outputId": "0967b031-86e9-4c68-9342-968f269ed4dc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 867
        }
      },
      "source": [
        "mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[-0.8367],\n",
              "         [ 0.1325],\n",
              "         [ 1.3578],\n",
              "         ...,\n",
              "         [-0.6594],\n",
              "         [-0.8688],\n",
              "         [ 1.5665]],\n",
              "\n",
              "        [[-0.2754],\n",
              "         [-1.1232],\n",
              "         [-0.0911],\n",
              "         ...,\n",
              "         [-0.9485],\n",
              "         [-1.5469],\n",
              "         [ 2.0030]],\n",
              "\n",
              "        [[-0.3617],\n",
              "         [-1.2107],\n",
              "         [-0.0183],\n",
              "         ...,\n",
              "         [-0.8000],\n",
              "         [-1.4827],\n",
              "         [ 2.1218]],\n",
              "\n",
              "        ...,\n",
              "\n",
              "        [[ 0.0589],\n",
              "         [-0.4579],\n",
              "         [ 0.0137],\n",
              "         ...,\n",
              "         [-0.7118],\n",
              "         [-1.5851],\n",
              "         [ 1.5854]],\n",
              "\n",
              "        [[ 0.0589],\n",
              "         [-0.4579],\n",
              "         [ 0.0137],\n",
              "         ...,\n",
              "         [-0.7118],\n",
              "         [-1.5851],\n",
              "         [ 1.5854]],\n",
              "\n",
              "        [[ 0.0589],\n",
              "         [-0.4579],\n",
              "         [ 0.0137],\n",
              "         ...,\n",
              "         [-0.7118],\n",
              "         [-1.5851],\n",
              "         [ 1.5854]]], grad_fn=<UnsqueezeBackward0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kP1fgC65MeeK"
      },
      "source": [
        "#mask = mask.unsqueeze(-1).float()\n",
        "out_masked = x * mask\n",
        "out_sum = out_masked.sum(dim=1)\n",
        "mask_sum = mask.sum(dim=(1))\n",
        "out_avg_pooling = out_sum / mask_sum"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GYNAIlejPkpp",
        "outputId": "b146c669-7186-4f4a-d2aa-e5bc8fde9cd0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "out_avg_pooling.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6UPE2p3KcrLa",
        "outputId": "a7263d13-a87d-42d8-9372-5a2b3300ed47",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "out_avg_pooling[2]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([-0.4213, -0.7786, -0.3129,  ..., -0.8124, -0.9138,  0.9624],\n",
              "       grad_fn=<SelectBackward>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 140
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CTnJoJUecmx4",
        "outputId": "a384c5d3-45fb-425a-afc4-0e8f507d533f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "out_avg_pooling[2]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([-0.4213, -0.7786, -0.3129,  ..., -0.8124, -0.9138,  0.9624],\n",
              "       grad_fn=<SelectBackward>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 137
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sIruQMxp4HpO",
        "outputId": "f086af47-a44a-4265-deb9-a145f4ef1e88",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 346
        }
      },
      "source": [
        "output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[[-1.5424,  0.2528,  1.4187,  ..., -0.7494, -0.6920,  1.3466],\n",
              "          [-0.1785, -0.5660, -0.4416,  ..., -0.8460, -0.5389,  2.2484],\n",
              "          [-0.1877, -1.0697, -0.4143,  ..., -0.9657, -0.4671,  2.0488],\n",
              "          ...,\n",
              "          [ 0.0614, -0.9423, -0.7553,  ..., -0.3587, -0.3514,  1.1257],\n",
              "          [ 0.0614, -0.9423, -0.7553,  ..., -0.3587, -0.3514,  1.1257],\n",
              "          [ 0.0614, -0.9423, -0.7553,  ..., -0.3587, -0.3514,  1.1257]],\n",
              " \n",
              "         [[-1.3017,  0.3100,  2.0458,  ..., -0.6810, -0.9355,  1.1852],\n",
              "          [ 0.0084, -1.3484, -0.0554,  ..., -0.8908, -1.5854,  2.2086],\n",
              "          [ 0.0089, -1.3423, -0.0841,  ..., -0.8990, -1.5419,  2.1772],\n",
              "          ...,\n",
              "          [-0.1941, -1.1582, -0.1775,  ..., -0.9945, -1.2986,  1.8684],\n",
              "          [-0.2826, -0.6239, -0.2529,  ..., -0.7868, -1.3955,  1.4536],\n",
              "          [-0.4169,  0.1849,  0.0186,  ..., -0.6474, -1.5506,  1.0303]]],\n",
              "        grad_fn=<AddBackward0>),\n",
              " tensor([[ True,  True,  True,  True,  True,  True,  True, False, False, False],\n",
              "         [ True,  True,  True,  True,  True,  True,  True,  True,  True,  True]]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vVHBoV8rR5Dm",
        "outputId": "3a8752da-ec7c-4ccd-fd5a-cf9a57fc2a9c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.9736],\n",
              "        [1.0840]], grad_fn=<AddmmBackward>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GSkztuAJ3UCf",
        "outputId": "9cb27fb5-05e2-41f6-d9e8-0f1485b8253f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(batch)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3IcvkLHGMZv8",
        "outputId": "16df4636-64d2-48d0-dab4-3fe5cc7865db",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(batch[1])\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3AsU_iXUb03q",
        "outputId": "60bd39de-f116-4ad8-e0c1-e6ac4ceab2a5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 188
        }
      },
      "source": [
        "batch[0][0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "        [0., 1., 1., 0., 0., 0., 0., 0., 0., 0.],\n",
              "        [0., 1., 1., 1., 0., 0., 0., 0., 0., 0.],\n",
              "        [0., 0., 1., 1., 1., 0., 0., 0., 1., 0.],\n",
              "        [0., 0., 0., 1., 1., 1., 0., 0., 0., 0.],\n",
              "        [0., 0., 0., 0., 1., 1., 1., 0., 0., 0.],\n",
              "        [0., 0., 0., 0., 0., 1., 1., 1., 0., 0.],\n",
              "        [0., 0., 0., 0., 0., 0., 1., 1., 1., 1.],\n",
              "        [0., 0., 0., 1., 0., 0., 0., 1., 1., 0.],\n",
              "        [0., 0., 0., 0., 0., 0., 0., 1., 0., 1.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7qxOTp1Sbuns"
      },
      "source": [
        "btbatch=batch[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3qfDB7lzK6ZG",
        "outputId": "ba3fea92-a641-410b-9d0e-2986a30d5fd8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(outputs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "11"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AmlVfAflLDVK",
        "outputId": "67d9db93-7567-4c5a-b6c9-9749898d83a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(outputs[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "njluxZSpLFcD",
        "outputId": "0a8585f4-55f5-4995-9b71-5c6b30f7c04f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 165
        }
      },
      "source": [
        "outputs[0][0].shape[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "IndexError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-91-e317b8112546>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m: tuple index out of range"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UCZnyUQNKm-j",
        "outputId": "aefb46ff-517f-4e42-f179-396f5e464f65",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "data_loader"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch.utils.data.dataloader.DataLoader at 0x7fd56c88ff60>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uFlrMjp95olj",
        "outputId": "e83fba52-1956-41d5-a466-71bed33df67a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "output.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X8DgKnvM5zqd",
        "outputId": "545eaf76-a3bf-415a-8e95-e054bbff3ec1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-0.2372],\n",
              "        [ 0.4631]], grad_fn=<AddmmBackward>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0RHA7nOI78n8",
        "outputId": "e725b807-77e6-42a8-af43-e69675355da3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(batch)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PzoNxCoc6Cxz"
      },
      "source": [
        "i=0\n",
        "for batch in data_loader:\n",
        "  #print(batch)\n",
        "  i=i+1\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5j17tFj-JhbW",
        "outputId": "9a683e52-1660-4e31-b983-d4cd749e1f62",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "i"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "11"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cajfGxt17T_e",
        "outputId": "2a3bb97d-bcbc-466d-af80-920b01dca4f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 188
        }
      },
      "source": [
        "\n",
        "batch[0][0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "        [0., 1., 1., 0., 0., 0., 0., 0., 0., 0.],\n",
              "        [0., 1., 1., 1., 0., 0., 0., 0., 0., 0.],\n",
              "        [0., 0., 1., 1., 1., 0., 0., 0., 1., 0.],\n",
              "        [0., 0., 0., 1., 1., 1., 0., 0., 0., 0.],\n",
              "        [0., 0., 0., 0., 1., 1., 1., 0., 0., 0.],\n",
              "        [0., 0., 0., 0., 0., 1., 1., 1., 0., 0.],\n",
              "        [0., 0., 0., 0., 0., 0., 1., 1., 1., 1.],\n",
              "        [0., 0., 0., 1., 0., 0., 0., 1., 1., 0.],\n",
              "        [0., 0., 0., 0., 0., 0., 0., 1., 0., 1.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K68wBjZw745n",
        "outputId": "8fcd817f-fb40-4d3c-e7b2-793b1962a7ac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "i"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "11"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "48IPKxy77ldh",
        "outputId": "5ab54bd3-43da-4e74-f8c4-628750b4365f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 451
        }
      },
      "source": [
        "batch[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 1., 1., 1., 0., 0., 0., 1., 0., 0., 0., 0.],\n",
              "         [0., 0., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 1., 1., 1., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 1., 1., 1., 0., 0., 1., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0.],\n",
              "         [0., 0., 1., 0., 0., 0., 1., 1., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 1., 1.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1.]],\n",
              "\n",
              "        [[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pj0NcLoe7Wv8",
        "outputId": "646096d6-0db4-4644-eed4-9c0849be07d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(batch)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x-110Qt87cV9",
        "outputId": "6da0a265-0097-4921-dc6d-3fdc2c2429b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(batch[0][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "12"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eo6p6sup7idc"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kGQjPuMfoH1U"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cRaMkilotu62"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}