BuildOwn Single HippoCell - Config.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using this notebook\n",
    " \n",
    "- You have to execute this notebook by running each cell consecutively.\n",
    "- To retain any changes made, you need to save the notebook. \n",
    "\n",
    "For detailed information about the notebook interface see _Help -> User Interface Tour_ in the menu, and the [Guidebook](https://lbologna.github.io/hbp-sp6-guidebook/getting_started/getting_started.html#working-with-collabs) for working with Collabs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import os\n",
    "import re\n",
    "import ast\n",
    "import sys\n",
    "import json\n",
    "import time\n",
    "import pytz\n",
    "import base64\n",
    "import numpy\n",
    "import neurom\n",
    "import pandas\n",
    "import pprint\n",
    "import shutil\n",
    "import logging\n",
    "import tarfile\n",
    "import zipfile\n",
    "import requests\n",
    "import warnings\n",
    "import datetime\n",
    "import matplotlib\n",
    "import unicodedata\n",
    "import xml.etree.ElementTree\n",
    "\n",
    "from neurom import viewer\n",
    "from ipywidgets.widgets import *\n",
    "from IPython.display import HTML,Javascript, display, clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.isdir('GUIBPOPT')==True:\n",
    "    shutil.rmtree('GUIBPOPT')\n",
    "if os.path.isdir('GUIBPOPT')==False:\n",
    "    !mkdir GUIBPOPT\n",
    "os.chdir(\"GUIBPOPT\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(HTML('''<style>\n",
    "    .widget-label { min-width: 17ex !important;}\n",
    "</style>'''))\n",
    "\n",
    "javascript2 = \"\"\"\n",
    "<script type=\"text/Javascript\">\n",
    "document.styleSheets[0].insertRule(\".widget-listbox.form-control { width: auto }\", 1);\n",
    "document.styleSheets[0].insertRule(\".form_login > label { width: 30% }\", 1);\n",
    "document.styleSheets[0].insertRule(\".form_login > label + input { width: 70% }\", 1);\n",
    "</script>\n",
    "\"\"\"\n",
    "h1=HTML(javascript2)\n",
    "h1.display='none'\n",
    "display(h1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "URL=('https://object.cscs.ch/v1/AUTH_c0a333ecf7c045809321ce9d9ecdfdea/hippocampus_optimization/bsp_data_repository-4.0.5_Python3.tar.gz')\n",
    "!wget \"{URL}\"\n",
    "!tar -zxvf bsp_data_repository-4.0.5_Python3.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "listfilenames=[]\n",
    "for filename in os.listdir(os.getcwd()):\n",
    "    listfilenames.append(filename)\n",
    "if 'Work-in-progress3.png' not in listfilenames:\n",
    "    shutil.copy2(os.path.join('..','Work-in-progress3.png'), 'Work-in-progress3.png')\n",
    "if 'unicore_api_Python3.ipynb' not in listfilenames:\n",
    "    shutil.copy2(os.path.join('..','unicore_api_Python3.ipynb'), 'unicore_api_Python3.ipynb')\n",
    "    \n",
    "diff = lambda l1,l2: [x for x in l1 if x not in l2]\n",
    "os.chdir('bsp_data_repository-4.0.5_Python3')\n",
    "os.chdir('optimizations')\n",
    "FD=diff(os.listdir('.'),['README'])\n",
    "\n",
    "for i in range(len(FD)):\n",
    "    shutil.copytree(FD[i], os.path.join('..', '..', FD[i]))\n",
    "    \n",
    "F1=[]\n",
    "F2=[]\n",
    "for i in range(len(FD)):\n",
    "    poz=[m.start() for m in re.finditer('_', FD[i])]\n",
    "    F2.append(FD[i][poz[0]+1:poz[1]])\n",
    "    F1.append(FD[i][poz[2]+1:poz[len(poz)-1]])\n",
    "os.chdir('../../..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run \"unicore_api_Python3.ipynb\"\n",
    "logger = logging.getLogger()\n",
    "logging.disable(logging.WARNING)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_to_html(plt):\n",
    "    buf = io.BytesIO()\n",
    "    plt.savefig(buf, format='png')\n",
    "    buf.seek(0)\n",
    "    return \"\"\"<img src='data:image/png;base64,{}'/>\"\"\".format(base64.b64encode(buf.getvalue()).decode('ascii'))\n",
    "\n",
    "F3=[]\n",
    "for i in range(len(F1)):\n",
    "    F3.append(F1[i]+' ('+F2[i]+')')\n",
    "\n",
    "w3 = Select(description='morphology', options=set(F3), layout=Layout(width='50%',height='100px'))\n",
    "display(w3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices = [i for i, x in enumerate(F1) if x == w3.value[0:w3.value.find(' ')]]\n",
    "FF = []\n",
    "for i in range(len(indices)):\n",
    "    FF.append(FD[indices[i]])\n",
    "os.chdir(\"GUIBPOPT\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveparamfile(_):\n",
    "    os.chdir(w4.value)\n",
    "    os.chdir(w4.value)\n",
    "    os.chdir('config')\n",
    "    with open('parameters.json', 'w') as f:\n",
    "        json.dump(ast.literal_eval(H2.value) ,f)\n",
    "    f.close()\n",
    "    os.chdir('..')\n",
    "    os.chdir('..')\n",
    "    os.chdir('..')\n",
    "\n",
    "def on_w3_clicked(change):\n",
    "    out1.layout.display='none'\n",
    "    %matplotlib agg\n",
    "    clear_output(wait=True)\n",
    "    w4.layout.display='none'\n",
    "    tabs.layout.display='none'\n",
    "    indices = [i for i, x in enumerate(F1) if x == w3.value[0:w3.value.find(' ')]]\n",
    "    FF=[]\n",
    "    for i in range(len(indices)):\n",
    "        FF.append(FD[indices[i]])\n",
    "    dirnew=sorted(FF)[0]\n",
    "    for files in os.listdir(dirnew):\n",
    "        if files.endswith('.zip'):\n",
    "            os.chdir(dirnew)\n",
    "            zip_ref = zipfile.ZipFile(files, 'r')\n",
    "            zip_ref.extractall()\n",
    "            zip_ref.close()          \n",
    "    os.chdir('..')\n",
    "    os.chdir(dirnew)\n",
    "    os.chdir(dirnew)\n",
    "    os.chdir('morphology')\n",
    "    for files in os.listdir('.'):\n",
    "        if files.endswith('.asc'):\n",
    "            nrn = neurom.load_neuron(w3.value[0:w3.value.find(' ')]+'.asc');\n",
    "        else:\n",
    "            if files.endswith('.ASC'):\n",
    "                nrn = neurom.load_neuron(w3.value[0:w3.value.find(' ')]+'.ASC');\n",
    "    plt,ax=viewer.draw(nrn, mode='2d');\n",
    "    for files in os.listdir('.'):\n",
    "        if files.endswith('.png'):\n",
    "            os.remove(files)\n",
    "    plt.savefig(\"{}.png\".format(w3.value))\n",
    "    plt.clear()\n",
    "    %matplotlib inline\n",
    "    file = open(\"{}.png\".format(w3.value), \"rb\")\n",
    "    image = file.read()\n",
    "    out1.value=image\n",
    "    out1.layout.display=''\n",
    "    os.chdir('..')\n",
    "    os.chdir('config')\n",
    "    with open('features.json') as data_file:    \n",
    "        data = json.load(data_file)\n",
    "    H1.value=pprint.pformat(data[list(data.keys())[0]],width=80,depth=5)\n",
    "    H1.disabled=True\n",
    "    with open('parameters.json') as data_file:    \n",
    "        data = json.load(data_file)\n",
    "    H2.value=pprint.pformat(data,width=120,depth=5)\n",
    "    os.chdir('..')\n",
    "    os.chdir('..')\n",
    "    os.chdir('..')\n",
    "    w4.options=sorted(FF)\n",
    "    w4.layout.display=''\n",
    "    tabs.selected_index=0\n",
    "    tabs.layout.display=''\n",
    "    \n",
    "def on_w4_clicked(change):\n",
    "    clear_output(wait=True)\n",
    "    os.chdir(w4.value)\n",
    "    if os.path.isdir(w4.value)==False:\n",
    "        dirloc=os.getcwd()\n",
    "        for files in os.listdir(dirloc):\n",
    "            if files.endswith('.zip'):\n",
    "                os.chdir(dirloc)\n",
    "                zip_ref = zipfile.ZipFile(files, 'r')\n",
    "                zip_ref.extractall()\n",
    "                zip_ref.close()  \n",
    "    os.chdir(w4.value)\n",
    "    os.chdir('config')\n",
    "    with open('features.json') as data_file:    \n",
    "        data = json.load(data_file)\n",
    "    H1.value=pprint.pformat(data[list(data.keys())[0]],width=80,depth=5)\n",
    "    H1.disabled=True\n",
    "    with open('parameters.json') as data_file:    \n",
    "        data = json.load(data_file)\n",
    "    H2.value=pprint.pformat(data,width=120,depth=5)\n",
    "    os.chdir('..')\n",
    "    os.chdir('..')\n",
    "    os.chdir('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w4 = Select(description='configuration',layout=Layout(width='50%',height='100px'))\n",
    "w4.options = sorted(FF)\n",
    "\n",
    "for files in os.listdir(w4.value):\n",
    "    if files.endswith('.zip'):\n",
    "        os.chdir(w4.value)\n",
    "        zip_ref = zipfile.ZipFile(files, 'r')\n",
    "        zip_ref.extractall()\n",
    "        zip_ref.close()      \n",
    "\n",
    "os.chdir('..')\n",
    "os.chdir(w4.value)\n",
    "os.chdir(w4.value)\n",
    "os.chdir('morphology')\n",
    "\n",
    "for files in os.listdir('.'):\n",
    "    if files.endswith('.asc'):\n",
    "        nrn = neurom.load_neuron(w3.value[0:w3.value.find(' ')]+'.asc');\n",
    "    else:\n",
    "        if files.endswith('.ASC'):\n",
    "            nrn = neurom.load_neuron(w3.value[0:w3.value.find(' ')]+'.ASC');\n",
    "\n",
    "%matplotlib agg\n",
    "plt, ax = viewer.draw(nrn, mode='2d');\n",
    "plt.savefig(\"{}.png\".format(w3.value))\n",
    "plt.clear()\n",
    "%matplotlib inline\n",
    "file = open(\"{}.png\".format(w3.value), \"rb\")\n",
    "image = file.read()\n",
    "out1 = Image(value=image,format='png',width=400,height=500,)\n",
    "display(out1)\n",
    "os.chdir('..')\n",
    "os.chdir('..')\n",
    "os.chdir('..')\n",
    "\n",
    "H1 = Textarea(layout=Layout(width='80%', height='200px'))\n",
    "H2 = Textarea(layout=Layout(width='80%', height='200px'))\n",
    "os.chdir(w4.value)\n",
    "os.chdir(w4.value)\n",
    "os.chdir('config')\n",
    "with open('features.json') as data_file:    \n",
    "    data = json.load(data_file)\n",
    "H1.value = pprint.pformat(data[list(data.keys())[0]],width=80,depth=5)\n",
    "H1.disabled = True\n",
    "with open('parameters.json') as data_file:    \n",
    "    data = json.load(data_file)\n",
    "H2.value = pprint.pformat(data,width=120,depth=5)\n",
    "os.chdir('..')\n",
    "os.chdir('..')\n",
    "os.chdir('..')\n",
    "BT1 = Button()\n",
    "BT1.description = 'Save new parameters'\n",
    "BT1.background_color = 'gainsboro'\n",
    "BT1.border_color = 'black'\n",
    "BT1.on_click(saveparamfile)\n",
    "\n",
    "page1 = Box(children = [w4])\n",
    "page2 = Box(children = [H1])\n",
    "page3 = Box(children = [VBox([H2, BT1], layout=Layout(width='100%'))])\n",
    "tabs = Tab(children = [page1,page2,page3])\n",
    "tabs.set_title(0, 'Configuration')\n",
    "tabs.set_title(1, 'Features')\n",
    "tabs.set_title(2, 'Parameters')\n",
    "display(tabs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createzip(_):\n",
    "    os.chdir(w4.value)\n",
    "    dirloc=os.getcwd()\n",
    "    for file in os.listdir(dirloc):\n",
    "        if file.startswith('init') or file.endswith('.zip'):\n",
    "            os.remove(file)\n",
    "    with open('init.py','w') as f:\n",
    "        f.write('import os')\n",
    "        f.write('\\n')\n",
    "        f.write('os.system(\\'python3 opt_neuron.py --max_ngen='+NGEN.value+' --offspring_size='+OS.value+\\\n",
    "                ' --start --checkpoint ./checkpoints/checkpoint.pkl\\')')\n",
    "        f.write('\\n')\n",
    "    f.close()\n",
    "    os.chdir(w4.value)\n",
    "    dirloc=os.getcwd()\n",
    "    for file in os.listdir(dirloc):\n",
    "        if file.startswith('init'):\n",
    "            os.remove(file)\n",
    "    os.chdir('..')\n",
    "    shutil.copy('init.py',os.path.join('.',w4.value,'init.py'))\n",
    "    global zfName\n",
    "    poz=[m.start() for m in re.finditer('_', w4.value)]\n",
    "    zfName = w4.value[0:poz[len(poz)-1]]+'_'+datetime.datetime.now().strftime('%Y%m%d%H%M%S')+'.zip'\n",
    "    shutil.copytree(w4.value,zfName[0:zfName.find('.zip')])\n",
    "    foldnameinside=w4.value #zfName[0:zfName.find('.zip')]\n",
    "    foo = zipfile.ZipFile(zfName, 'w')\n",
    "    for root, dirs, files in os.walk(os.path.join('.',foldnameinside)):\n",
    "        if (root==os.path.join('.',foldnameinside,'morphology')) or \\\n",
    "        (root==os.path.join('.',foldnameinside,'config')) or \\\n",
    "        (root==os.path.join('.',foldnameinside,'mechanisms')) or \\\n",
    "        (root==os.path.join('.',foldnameinside,'model')):\n",
    "            for f in files:\n",
    "                foo.write(os.path.join(root, f))\n",
    "        if (root==os.path.join('.',foldnameinside,'checkpoints')) or \\\n",
    "        (root==os.path.join('.',foldnameinside,'figures')):\n",
    "            foo.write(os.path.join(root))\n",
    "        if (root==os.path.join('.',foldnameinside)):\n",
    "            for f in files:\n",
    "                if f.endswith('.py'):\n",
    "                    foo.write(os.path.join(root, f))                    \n",
    "    foo.close()\n",
    "    os.chdir('..')\n",
    "    BT.disabled=True\n",
    "    \n",
    "    \n",
    "def createzipPizDaint(_):\n",
    "    os.chdir(w4.value)\n",
    "    dirloc=os.getcwd()\n",
    "    for file in os.listdir(dirloc):\n",
    "        if file.startswith(\"ipyparallel.sbatch\") or file.startswith(\"zipfolder.py\"):\n",
    "            os.remove(file)\n",
    "    with open('zipfolder.py','w') as f:\n",
    "        f.write('import os\\n')\n",
    "        f.write('import zipfile\\n')\n",
    "        f.write('retval = os.getcwd()\\n')\n",
    "        f.write('print(\"Current working directory %s\" % retval)\\n')\n",
    "        f.write('os.chdir(\\'..\\')\\n')\n",
    "        f.write('retval = os.getcwd()\\n')\n",
    "        f.write('print(\"Current working directory %s\" % retval)\\n')\n",
    "        f.write('def zipdir(path, ziph):\\n')\n",
    "        f.write('    for root, dirs, files in os.walk(path):\\n')\n",
    "        f.write('        for file in files:\\n')\n",
    "        f.write('            ziph.write(os.path.join(root, file))\\n')\n",
    "        f.write('zipf = zipfile.ZipFile(\\'output.zip\\', \\'w\\')\\n')\n",
    "        f.write('zipdir(\\''+w4.value+'/\\', zipf)\\n')\n",
    "    f.close()\n",
    "    with open('ipyparallel.sbatch','w') as f:\n",
    "        f.write('#!/bin/bash -l\\n')\n",
    "        f.write('\\n')\n",
    "        f.write('mkdir logs\\n')\n",
    "        f.write('#SBATCH --job-name=bluepyopt_ipyparallel\\n')\n",
    "        f.write('#SBATCH --error=logs/ipyparallel_%j.log\\n')\n",
    "        f.write('#SBATCH --output=logs/ipyparallel_%j.log\\n')\n",
    "        f.write('#SBATCH --partition=normal\\n')\n",
    "        f.write('#SBATCH --constraint=mc\\n')\n",
    "        f.write('\\n')\n",
    "        f.write('export PMI_NO_FORK=1\\n')\n",
    "        f.write('export PMI_NO_PREINITIALIZE=1\\n')\n",
    "        f.write('export PMI_MMAP_SYNC_WAIT_TIME=300 \\n')\n",
    "        f.write('set -e\\n')\n",
    "        f.write('set -x\\n')\n",
    "        f.write('\\n')\n",
    "        f.write('module load daint-mc cray-python/3.8.2.1 PyExtensions/python3-CrayGNU-20.11\\n')\n",
    "        f.write(\"module load intel/19.0.1.144\\n\")\n",
    "        f.write('module use /apps/hbp/ich002/hbp-spack-deployments/softwares/27-10-2020/install/modules/tcl/cray-cnl7-haswell\\n')\n",
    "        f.write('module load neuron/8.0a\\n')\n",
    "        f.write('module load py-bluepyopt\\n')\n",
    "        f.write('nrnivmodl mechanisms\\n')\n",
    "        f.write('\\n')\n",
    "        f.write('export USEIPYP=1\\n')\n",
    "        f.write('export IPYTHONDIR=\"`pwd`/.ipython\"\\n')\n",
    "        f.write('export IPYTHON_PROFILE=ipyparallel.${SLURM_JOBID}\\n')\n",
    "        f.write('ipcontroller --init --sqlitedb --ip=\\'*\\' --profile=${IPYTHON_PROFILE} &\\n')\n",
    "        f.write('sleep 30\\n')\n",
    "        f.write('srun ipengine --profile=${IPYTHON_PROFILE} &\\n')\n",
    "        f.write('CHECKPOINTS_DIR=\"checkpoints\"\\n')\n",
    "        f.write('python opt_neuron.py --offspring_size='+OS.value+\\\n",
    "                ' --max_ngen='+NGEN.value+' --start --checkpoint \"${CHECKPOINTS_DIR}/checkpoint.pkl\"\\n') \n",
    "        f.write('python zipfolder.py')\n",
    "        f.write('\\n')\n",
    "    f.close()\n",
    "    os.chdir(w4.value)\n",
    "    dirloc=os.getcwd()\n",
    "    for file in os.listdir(dirloc):\n",
    "        if file.startswith('ipyparallel.sbatch') or file.startswith(\"zipfolder.py\"):\n",
    "            os.remove(file)\n",
    "    os.chdir('..')\n",
    "    shutil.copy('ipyparallel.sbatch',os.path.join('.', w4.value, 'ipyparallel.sbatch'))\n",
    "    shutil.copy('zipfolder.py',os.path.join('.', w4.value, 'zipfolder.py'))\n",
    "    global zfNamePiz\n",
    "    poz = [m.start() for m in re.finditer('_', w4.value)]\n",
    "    zfNamePiz = w4.value[0:poz[len(poz)-1]]+'_'+datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '.zip'\n",
    "    shutil.copytree(w4.value,zfNamePiz[0:zfNamePiz.find('.zip')])\n",
    "    foldnameinside=w4.value #zfNamePiz[0:zfNamePiz.find('.zip')]\n",
    "    foo = zipfile.ZipFile(zfNamePiz, 'w')\n",
    "    for root, dirs, files in os.walk(os.path.join('.',foldnameinside)):\n",
    "        if (root==os.path.join('.',foldnameinside,'morphology')) or \\\n",
    "        (root==os.path.join('.',foldnameinside,'config')) or \\\n",
    "        (root==os.path.join('.',foldnameinside,'mechanisms')) or \\\n",
    "        (root==os.path.join('.',foldnameinside,'model')):\n",
    "            for f in files:\n",
    "                foo.write(os.path.join(root, f))\n",
    "        if (root==os.path.join('.',foldnameinside,'checkpoints')) or \\\n",
    "        (root==os.path.join('.',foldnameinside,'figures')):\n",
    "            foo.write(os.path.join(root))\n",
    "        if (root==os.path.join('.',foldnameinside)):\n",
    "            for f in files:\n",
    "                if f.endswith('.py') or f.endswith('.sbatch'):\n",
    "                    foo.write(os.path.join(root, f))                    \n",
    "    foo.close()\n",
    "    os.chdir('..')\n",
    "    H2.disabled=True\n",
    "    BT1.disabled=True\n",
    "    NGEN.disabled=True\n",
    "    OS.disabled=True\n",
    "    BTCSCS.disabled=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w3.observe(on_w3_clicked,names='value')\n",
    "w4.observe(on_w4_clicked,names='value')\n",
    "\n",
    "NGEN=Text(**dict(\n",
    "        margin='0px 0px 0px 16px'\n",
    "))\n",
    "NGEN.description='max_ngen'\n",
    "NGEN.value='2'\n",
    "NGEN.width=600\n",
    "display(NGEN)\n",
    "OS=Text(**dict(\n",
    "        margin='0px 0px 0px 16px'\n",
    "))\n",
    "OS.description='offspring_size'\n",
    "OS.value='10'\n",
    "OS.width=200\n",
    "display(OS)\n",
    "zfName=''\n",
    "    \n",
    "BT = Button(description = 'Create zip file', layout=Layout(width='40%'))\n",
    "BT.background_color='gainsboro'\n",
    "BT.border_color='black'\n",
    "BT.width='20%'\n",
    "BT.on_click(createzip)\n",
    "BTCSCS = Button(description = 'Create Piz Daint zip file', layout=Layout(width='40%'))\n",
    "BTCSCS.background_color='gainsboro'\n",
    "BTCSCS.border_color='black'\n",
    "BTCSCS.width='20%'\n",
    "BTCSCS.on_click(createzipPizDaint)\n",
    "\n",
    "logging.captureWarnings(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "username=''\n",
    "\n",
    "def checkloginNSG(_):\n",
    "    username=username_widget.value\n",
    "    password=password_widget.value\n",
    "    KEY = 'Application_Fitting-DA5A3D2F8B9B4A5D964D4D2285A49C57'\n",
    "    headers = {'cipres-appkey' : KEY}\n",
    "    URL = 'https://nsgr.sdsc.edu:8443/cipresrest/v1/job/'+username\n",
    "    r = requests.get(URL, auth=(username, password),headers=headers,verify=False)\n",
    "    root = xml.etree.ElementTree.fromstring(r.text)\n",
    "    a = 0\n",
    "    for child in root:\n",
    "        if child.tag == 'displayMessage':\n",
    "            a = 1\n",
    "            msg = child.text\n",
    "    if a == 0:\n",
    "        msg = 'Authenticated successfully'\n",
    "        msg_widget.disabled=True\n",
    "        buttonlogin.disabled=True\n",
    "        username_widget.disabled=True\n",
    "        password_widget.disabled=True\n",
    "    msg_widget.value=msg\n",
    "    msg_widget.layout.display=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "username_widget = Text(description='Username:')\n",
    "username_widget.layout.width='37.5%'\n",
    "password_widget = Password(description='Password:')\n",
    "buttonlogin = Button()\n",
    "buttonlogin.description = 'Login NSG'\n",
    "buttonlogin.background_color='gainsboro'\n",
    "buttonlogin.border_color='black'\n",
    "buttonlogin.width='20%'\n",
    "buttonlogin.on_click(checkloginNSG)\n",
    "msg_widget = Text()\n",
    "msg_widget.layout.width='40%'\n",
    "msg_widget.layout.display='none'\n",
    "formMNSG = VBox(children=[username_widget,password_widget,buttonlogin,msg_widget])\n",
    "formMNSG.layout.display=''\n",
    "\n",
    "NNODES = Text()\n",
    "NNODES.description='number nodes'\n",
    "NNODES.value='2'\n",
    "NNODES.width=190\n",
    "NCORES = Text()\n",
    "NCORES.description='number cores'\n",
    "NCORES.value='10'\n",
    "NCORES.width=190\n",
    "RT = Text()\n",
    "RT.description='runtime'\n",
    "RT.value='1'\n",
    "RT.width=190\n",
    "file = open(\"Work-in-progress3.png\", \"rb\")\n",
    "image = file.read()\n",
    "IN = Image(value=image, format='png', width=400)\n",
    "IN.layout.display='none'\n",
    "STN = Text()\n",
    "STN.description='status'\n",
    "STN.value=''\n",
    "STN.disabled=True\n",
    "STN.width=75\n",
    "STN.layout.display='none'\n",
    "\n",
    "TT = Text()\n",
    "TT.description='Title:'\n",
    "Project=Text()\n",
    "Project.description='proj'\n",
    "Project.value=' '\n",
    "Project.width=190\n",
    "NNODESCSCS = Text()\n",
    "NNODESCSCS.description='number nodes'\n",
    "NNODESCSCS.value='6'\n",
    "NNODESCSCS.width=190\n",
    "NCORESCSCS = Text()\n",
    "NCORESCSCS.description='number cores'\n",
    "NCORESCSCS.value='24'\n",
    "NCORESCSCS.width=190\n",
    "RTCSCS = Text()\n",
    "RTCSCS.description='runtime'\n",
    "RTCSCS.value='120m'\n",
    "RTCSCS.width=190\n",
    "file = open(\"Work-in-progress3.png\", \"rb\")\n",
    "image = file.read()\n",
    "INCSCS = Image(value=image, format='png', width=400)\n",
    "INCSCS.layout.display='none'\n",
    "STNCSCS = Text()\n",
    "STNCSCS.description='status'\n",
    "STNCSCS.value=''\n",
    "STNCSCS.disabled=True\n",
    "STNCSCS.width=75\n",
    "STNCSCS.layout.display='none'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def runNSG(_):\n",
    "    myboxUNSG.children[0].disabled=True\n",
    "    IN.layout.display=''\n",
    "    os.chdir(w4.value)\n",
    "    \n",
    "    global CRA_USER, PASSWORD, KEY, URL, TOOL\n",
    "    username = username_widget.value\n",
    "    password = password_widget.value\n",
    "    CRA_USER = username\n",
    "    PASSWORD = password\n",
    "    KEY = 'Application_Fitting-DA5A3D2F8B9B4A5D964D4D2285A49C57'\n",
    "    URL = 'https://nsgr.sdsc.edu:8443/cipresrest/v1'\n",
    "    TOOL = 'BLUEPYOPT_EXPANSE'\n",
    "    \n",
    "    os.popen('pwd').read()\n",
    "    os.popen('ls -ltr').read()\n",
    "    headers = {'cipres-appkey' : KEY}\n",
    "    payload = {'tool' : TOOL, 'metadata.statusEmail' : 'false', 'vparam.number_cores_' : \\\n",
    "               unicodedata.normalize('NFKD', NCORES.value).encode('ascii','ignore'), \\\n",
    "               'vparam.number_nodes_' : unicodedata.normalize('NFKD', NNODES.value).encode('ascii','ignore'), \\\n",
    "               'vparam.runtime_' : unicodedata.normalize('NFKD', RT.value).encode('ascii','ignore'), \\\n",
    "               'vparam.filename_': 'init.py'}\n",
    "    files = {'input.infile_' : open(zfName,'rb')}\n",
    "\n",
    "    r = requests.post('{}/job/{}'.format(URL, CRA_USER), auth=(CRA_USER, PASSWORD), data=payload,\n",
    "                      headers=headers, files=files, verify=False)\n",
    "    \n",
    "    root = xml.etree.ElementTree.fromstring(r.text)\n",
    "\n",
    "    global outputuri, selfuri\n",
    "    for child in root:\n",
    "        if child.tag == 'resultsUri':\n",
    "            for urlchild in child:\n",
    "                if urlchild.tag == 'url':\n",
    "                    outputuri = urlchild.text\n",
    "        if child.tag == 'selfUri':\n",
    "            for urlchild in child:\n",
    "                if urlchild.tag == 'url':\n",
    "                    selfuri = urlchild.text\n",
    "\n",
    "    r = requests.get(selfuri, auth=(CRA_USER, PASSWORD), headers=headers,verify=False)\n",
    "    root = xml.etree.ElementTree.fromstring(r.text)\n",
    "    for child in root:\n",
    "        if child.tag=='jobHandle':\n",
    "            jobname=child.text\n",
    "\n",
    "    access_token = clb_oauth.get_token()\n",
    "    headers2 = {'Authorization': 'Bearer {}'.format(access_token)}\n",
    "    OIDC_OP_USER_ENDPOINT = \"https://iam.ebrains.eu/auth/realms/hbp/protocol/openid-connect/userinfo\"\n",
    "    res = requests.get(url=OIDC_OP_USER_ENDPOINT, headers=headers2)\n",
    "    filetoupload = str(\"listjobsNSG_\" + res.json()['preferred_username'] + \".txt\") \n",
    "    with open(os.path.join(\"..\", \"..\", filetoupload), \"a\") as myfile:\n",
    "        myfile.write(jobname)\n",
    "        myfile.write(' ')\n",
    "        myfile.write(zfName[0:zfName.find('.zip')])\n",
    "        myfile.write('\\n')\n",
    "    myfile.close()\n",
    "    myboxUNSG.children[1].layout.display = ''\n",
    "    os.chdir('..')\n",
    "\n",
    "    \n",
    "def runNSGifusernameandconfig(_):\n",
    "    username = username_widget.value\n",
    "    password = password_widget.value\n",
    "    if username != '':\n",
    "        if zfName != '':\n",
    "            runNSG(_)\n",
    "\n",
    "        \n",
    "def checksimnsg(CheckSNSG):\n",
    "    os.chdir(w4.value)\n",
    "    KEY = 'Application_Fitting-DA5A3D2F8B9B4A5D964D4D2285A49C57'\n",
    "    headers = {'cipres-appkey' : KEY}\n",
    "    r = requests.get(selfuri, auth=(CRA_USER, PASSWORD), headers=headers,verify=False)\n",
    "    root = xml.etree.ElementTree.fromstring(r.text)\n",
    "    for child in root:\n",
    "        if child.tag == 'jobStage':\n",
    "            jobtodisp=child.text\n",
    "        if child.tag == 'terminalStage':\n",
    "            jobstatus = child.text\n",
    "    STN.layout.display=''\n",
    "    STN.value=jobtodisp\n",
    "    if jobstatus == 'true':\n",
    "        storeto_path1 = os.path.join(\"..\", \"..\", \"BluePyOptAll\")\n",
    "        if os.path.isdir(storeto_path1) == False:\n",
    "            os.mkdir(storeto_path1)\n",
    "        os.chdir(storeto_path1)\n",
    "        # want a unique dir for each job run\n",
    "        storeto_path2 = os.path.join(\"resultsNSG\")\n",
    "        if os.path.isdir(storeto_path2) == False:\n",
    "            os.mkdir(storeto_path2)\n",
    "        os.chdir(storeto_path2)\n",
    "        \n",
    "        access_token = clb_oauth.get_token()\n",
    "        headers2 = {'Authorization': 'Bearer {}'.format(access_token)}\n",
    "        OIDC_OP_USER_ENDPOINT = \"https://iam.ebrains.eu/auth/realms/hbp/protocol/openid-connect/userinfo\"\n",
    "        res = requests.get(url=OIDC_OP_USER_ENDPOINT, headers=headers2)\n",
    "        \n",
    "        storeto_path3 = str(os.path.join(res.json()['preferred_username']))\n",
    "        if os.path.isdir(storeto_path3) == False:\n",
    "            os.mkdir(storeto_path3)\n",
    "        os.chdir(storeto_path3)\n",
    "        \n",
    "        storeto = os.path.join(zfName[0:zfName.find('.zip')])\n",
    "        os.mkdir(storeto)\n",
    "        os.chdir(storeto)\n",
    "        globaldownloadurilist = []\n",
    "        r = requests.get(outputuri,headers=headers, auth=(CRA_USER, PASSWORD),verify=False)\n",
    "        globaldownloadurilist = []\n",
    "        root = xml.etree.ElementTree.fromstring(r.text)\n",
    "        for child in root:\n",
    "            if child.tag == 'jobfiles':\n",
    "                for jobchild in child:\n",
    "                    if jobchild.tag == 'jobfile':\n",
    "                        for downloadchild in jobchild:\n",
    "                            if downloadchild.tag == 'downloadUri':\n",
    "                                for attchild in downloadchild:\n",
    "                                    if attchild.tag == 'url':\n",
    "                                        globaldownloadurilist.append(attchild.text)                              \n",
    "        # download all output files\n",
    "        for downloaduri in globaldownloadurilist:\n",
    "            r = requests.get(downloaduri, auth=(CRA_USER, PASSWORD), headers=headers, verify=False)\n",
    "            d = r.headers['content-disposition']\n",
    "            filename_list = re.findall('filename=(.+)', d)\n",
    "            for filename in filename_list:\n",
    "                with open(filename, 'wb') as fd:\n",
    "                    for chunk in r.iter_content():\n",
    "                        fd.write(chunk)\n",
    "        os.chdir('../../../../GUIBPOPT')\n",
    "        STN.layout.display='none'\n",
    "        IN.layout.display='none'\n",
    "        CheckSNSG.layout.display='none'\n",
    "        runningUNSG=False\n",
    "        myboxUNSG.children[0].disabled=False\n",
    "    else:\n",
    "        os.chdir('..')\n",
    "\n",
    "    \n",
    "def rununicoreCSCS(_):\n",
    "    RunSCSCS.disabled=True\n",
    "    INCSCS.layout.display=''\n",
    "    os.chdir(w4.value)\n",
    "    with open(zfNamePiz, 'rb') as content_file:\n",
    "        content = content_file.read()\n",
    "    fjob1 = {'To': zfNamePiz, 'Data':content}\n",
    "    inputs = [fjob1]\n",
    "    if TT.value=='':\n",
    "        jobname='UNICORE_Job'\n",
    "    else:\n",
    "        jobname=TT.value\n",
    "    job = {}\n",
    "    job = {\"Executable\": \"unzip \"+zfNamePiz+\"; cd \"+w4.value+\"; chmod +rx *.sbatch; ./ipyparallel.sbatch\",\n",
    "           \"Name\": jobname,\n",
    "           \"Resources\": {\"Nodes\": NNODESCSCS.value,\n",
    "                         \"CPUsPerNode\": NCORESCSCS.value,\n",
    "                         \"Runtime\": RTCSCS.value,\n",
    "                         #\"Memory\": \"4G\",\\\n",
    "                         \"NodeConstraints\" : \"mc\",\n",
    "                         \"Project\": Project.value,\n",
    "                        }, \n",
    "          }\n",
    "    auth = get_oidc_auth() \n",
    "    base_url = get_sites()['DAINT-CSCS']['url']\n",
    "    global job_url\n",
    "    job_url = submit(base_url + '/jobs', job, auth, inputs)\n",
    "    if job_url!='':\n",
    "        myboxUCSCS.children[1].layout.display=''\n",
    "    else:\n",
    "        INCSCS.layout.display='none'\n",
    "        myboxUCSCS.children[0].disabled=False\n",
    "    os.chdir('..')\n",
    "\n",
    "    \n",
    "def assign_file_types(filename):\n",
    "    \"\"\" assign file types based on file name and extension \"\"\"\n",
    "    fname, extension = os.path.splitext(filename)\n",
    "    if extension == '.txt':\n",
    "        filetype = 'text/plain'\n",
    "    else:\n",
    "        filetype = 'application/unknown'\n",
    "    return filetype\n",
    "\n",
    "\n",
    "def checksimCSCS(CheckCSCS):\n",
    "    RunSCSCS.disabled=True\n",
    "    auth = get_oidc_auth() \n",
    "    job_properties = get_properties(job_url, auth)\n",
    "    STNCSCS.layout.display=''\n",
    "    STNCSCS.value=job_properties['status']\n",
    "    if STNCSCS.value=='SUCCESSFUL':\n",
    "        auth = get_oidc_auth()\n",
    "        wd = get_working_directory(job_url, auth)\n",
    "        output_files = list_files(wd, auth)\n",
    "        storeto_path = os.path.join(\"..\", 'results')\n",
    "        if os.path.isdir(storeto_path)==False:\n",
    "            os.mkdir(storeto_path)\n",
    "        os.chdir(storeto_path)\n",
    "        storeto = str(os.path.join(storeto_path,'output_'+job_properties['submissionTime']))\n",
    "        os.mkdir(storeto)\n",
    "        os.chdir(storeto)\n",
    "        for file_path in output_files:\n",
    "            _, f = os.path.split(file_path)\n",
    "            if (f=='stderr') or (f=='stdout') or (f=='output.zip'):\n",
    "                filetype = assign_file_types(f)\n",
    "                content = get_file_content(wd+\"/files\"+file_path,auth)\n",
    "                with open(f,\"wb\") as local_file:\n",
    "                    local_file.write(content)\n",
    "        os.chdir('../../GUIBPOPT')\n",
    "        STNCSCS.layout.display='none'\n",
    "        INCSCS.layout.display='none'\n",
    "        CheckSCSCS.layout.display='none'\n",
    "        myboxUCSCS.children[1].layout.display='none'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =============================================== SERVICE ACCOUNT ======================================================       \n",
    "def run_on_service_account_pizdaint(_):\n",
    "    box_SA.children[0].disabled = True\n",
    "    INCSCS.layout.display=''\n",
    "    \n",
    "    SUBMIT_URL = 'https://bspsa.cineca.it/jobs/pizdaint/' \n",
    "    \n",
    "    command = \"unzip \"+zfNamePiz+\"; cd \" + w4.value + \"; chmod +rx *.sbatch; ./ipyparallel.sbatch\"\n",
    "    payload = {\n",
    "        'command': command, \n",
    "        'node_number': str(NNODESCSCS.value),\n",
    "        'core_number': str(NCORESCSCS.value),\n",
    "        'runtime': float(runtimesa.value)\n",
    "    }\n",
    "    \n",
    "    os.chdir(w4.value)\n",
    "    job_file = {'file': open(zfNamePiz, 'rb')}\n",
    "    headers = get_oidc_auth()\n",
    "    headers.update({'Content-Disposition': 'attachment;filename=' + zfNamePiz})\n",
    "    headers.update({'payload': json.dumps(payload)})\n",
    "    \n",
    "    r = requests.post(url=SUBMIT_URL, headers=headers, files=job_file)\n",
    "    if r.status_code == 201:\n",
    "        job = r.json()\n",
    "        global jobid\n",
    "        jobid = job['job_id']\n",
    "        box_SA.children[1].layout.display = ''\n",
    "    else:\n",
    "        INCSCS.layout.display = 'none'\n",
    "        box_SA.children[0].disabled = False\n",
    "    os.chdir(\"..\")\n",
    "        \n",
    "\n",
    "def check_on_service_account_pizdaint(_):\n",
    "    box_SA.children[1].disabled = True\n",
    "    CHECK_JOB_URL = 'https://bspsa.cineca.it/jobs/pizdaint/bsp_pizdaint_01/' + jobid + '/'\n",
    "    FILES_URL = 'https://bspsa.cineca.it/files/pizdaint/bsp_pizdaint_01/'\n",
    "    \n",
    "    headers = get_oidc_auth() \n",
    "    \n",
    "    r = requests.get(url=CHECK_JOB_URL, headers=headers)\n",
    "    if r.status_code == 200:\n",
    "        job = r.json()\n",
    "                \n",
    "        STNCSCS.layout.display=''\n",
    "        STNCSCS.value = job['stage']    \n",
    "\n",
    "        if job['stage'] == 'SUCCESSFUL':\n",
    "            r = requests.get(url=FILES_URL + jobid + '/', headers=headers)\n",
    "            if r.status_code == 200:\n",
    "                files_list = r.json()\n",
    "                storeto_path = os.path.join('../resultsPizdaint-SA')\n",
    "                if os.path.isdir(storeto_path)==False:\n",
    "                    os.mkdir(storeto_path)\n",
    "                os.chdir(storeto_path)\n",
    "                storeto = str('output_'+job['init_date'])\n",
    "                os.mkdir(storeto)\n",
    "                os.chdir(storeto)\n",
    "                \n",
    "                for f in files_list:\n",
    "                    if f == '/stderr' or f == '/stdout' or f == '/output.zip':\n",
    "                        filename = f[1:]\n",
    "                        file_type = assign_file_types(f)\n",
    "                        DOWNLOAD_FILE_URL = FILES_URL + jobid + f + '/'\n",
    "                        r = requests.get(url=DOWNLOAD_FILE_URL, headers=headers)\n",
    "                        if r.status_code == 200:\n",
    "                            with open(filename, 'wb') as local_file:\n",
    "                                local_file.write(r.content)\n",
    "                \n",
    "                os.chdir('../../GUIBPOPT')\n",
    "                STNCSCS.layout.display = 'none'\n",
    "                INCSCS.layout.display = 'none'\n",
    "                checksa_button.layout.display = 'none'\n",
    "                box_SA.children[1].layout.display = 'none'\n",
    "                \n",
    "        else:\n",
    "            box_SA.children[1].disabled = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =======================================================================================================================\n",
    "        \n",
    "RunSNSG = Button(description = 'Run NSG simulation', layout=Layout(width='40%'))\n",
    "RunSNSG.background_color='gainsboro'\n",
    "RunSNSG.border_color='black'\n",
    "RunSNSG.width='20%'\n",
    "RunSNSG.on_click(runNSGifusernameandconfig)\n",
    "CheckSNSG = Button(description = 'Check NSG simulation', layout=Layout(width='25%'))\n",
    "CheckSNSG.background_color='gainsboro'\n",
    "CheckSNSG.border_color='black'\n",
    "CheckSNSG.on_click(checksimnsg)\n",
    "buttonSNSG =[RunSNSG, CheckSNSG]\n",
    "myboxUNSG =HBox(children=buttonSNSG)\n",
    "myboxUNSG.children[0].layout.display=''\n",
    "myboxUNSG.children[1].layout.display='none'\n",
    "\n",
    "RunSCSCS = Button(description = 'Run PizDaint simulation', layout=Layout(width='40%'))\n",
    "RunSCSCS.background_color='gainsboro'\n",
    "RunSCSCS.border_color='black'\n",
    "RunSCSCS.on_click(rununicoreCSCS)\n",
    "CheckSCSCS = Button(description = 'Check PizDaint simulation', layout=Layout(width='40%'))\n",
    "CheckSCSCS.background_color='gainsboro'\n",
    "CheckSCSCS.border_color='black'\n",
    "CheckSCSCS.on_click(checksimCSCS)\n",
    "buttonSCSCS = [RunSCSCS, CheckSCSCS]\n",
    "myboxUCSCS = HBox(children=buttonSCSCS)\n",
    "myboxUCSCS.children[0].layout.display=''\n",
    "myboxUCSCS.children[1].layout.display='none'\n",
    "\n",
    "# ======================================== SERVICE ACCOUNT Accordion Button ============================================\n",
    "runtimesa = Text(description='Runtime')\n",
    "runtimesa.value='2.0'\n",
    "runtimesa.width=190\n",
    "runsa_button = Button(description='Run Service Account/PizDaint simulation', layout=Layout(width='40%'))\n",
    "runsa_button.background_color = 'gainsboro'\n",
    "runsa_button.border_color = 'black'\n",
    "runsa_button.on_click(run_on_service_account_pizdaint)\n",
    "checksa_button = Button(description='Check Service Account/PizDaint simulation', layout=Layout(width='40%'))\n",
    "checksa_button.background_color = 'gainsboro'\n",
    "checksa_button.border_color = 'black'\n",
    "checksa_button.layout.display = 'none'\n",
    "checksa_button.on_click(check_on_service_account_pizdaint)\n",
    "box_SA = VBox(children=[runsa_button, checksa_button])\n",
    "# ======================================================================================================================\n",
    "\n",
    "accordion = Accordion(children=[VBox([formMNSG, NNODES, NCORES, RT, STN, IN, BT, myboxUNSG]),\n",
    "                                VBox([TT, Project, NNODESCSCS, NCORESCSCS, RTCSCS, STNCSCS, INCSCS, BTCSCS, myboxUCSCS], \n",
    "                                     layout=Layout(width='100%')),\n",
    "                                VBox([NNODESCSCS, NCORESCSCS, runtimesa, STNCSCS, INCSCS, BTCSCS, box_SA], \n",
    "                                     layout=Layout(width='100%'))])\n",
    "accordion.set_title(0, 'NSG')\n",
    "accordion.set_title(1, 'PizDaint')\n",
    "accordion.set_title(2, 'Service Account - Pizdaint')\n",
    "accordion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import HTML\n",
    "\n",
    "HTML('''\n",
    "<button class=\"toggle_button\" onclick=\"toggleCode(this)\">Show Code</button>\n",
    "<script>\n",
    "    function toggleCode(button) {\n",
    "        boxes = button.closest(\".jp-Notebook\").querySelectorAll(\".jp-Cell-inputWrapper\");\n",
    "        if (boxes[0].style.display == \"none\") {\n",
    "            boxes.forEach(box => box.style.display = '');\n",
    "            button.innerHTML = \"Hide Code\";\n",
    "        } else {\n",
    "            boxes.forEach(box => box.style.display = 'none');\n",
    "            button.innerHTML = \"Show Code\";\n",
    "        }\n",
    "    }\n",
    "    document.querySelectorAll(\".jp-Cell-inputWrapper\").forEach(box => box.style.display = 'none');\n",
    "</script>\n",
    "''')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
  },
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": ""
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}