KWS_OURS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ea2b1ff4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from finn.util.basic import make_build_dir\n",
    "from finn.util.visualization import showInNetron\n",
    "    \n",
    "build_dir = \"/workspace/finn/0928\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b48720c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import onnx\n",
    "from finn.util.test import get_test_model_trained\n",
    "import brevitas.onnx as bo\n",
    "from finn.core.modelwrapper import ModelWrapper\n",
    "from finn.transformation.infer_shapes import InferShapes\n",
    "from finn.transformation.fold_constants import FoldConstants\n",
    "from finn.transformation.general import GiveReadableTensorNames, GiveUniqueNodeNames, RemoveStaticGraphInputs\n",
    "\n",
    "'''#kws = get_test_model_trained(\"KWS\", 1, 1)\n",
    "#bo.export_finn_onnx(kws, (1, 16, 16000, 1), build_dir + \"/ckpt.t7.M5_11111.pth.finn.onnx\")\n",
    "model = ModelWrapper(build_dir + \"/ckpt.t7.M5_11111_0902_2.pth.finn.onnx\")\n",
    "model = model.transform(InferShapes())\n",
    "model = model.transform(FoldConstants())\n",
    "model = model.transform(GiveUniqueNodeNames())\n",
    "model = model.transform(GiveReadableTensorNames())\n",
    "model = model.transform(RemoveStaticGraphInputs())\n",
    "model.save(build_dir + \"/ckpt.t7.M5_11111.pth.finn_tidy.onnx\")'''\n",
    "model = ModelWrapper(build_dir + \"/ckpt.t7.M5_11111.pth.tidy.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "32019d46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.tidy.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7faa63e6a9d0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir + \"/ckpt.t7.M5_11111.pth.tidy.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f908a94f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/workspace/finn-base/src/finn/transformation/infer_data_layouts.py:114: UserWarning: Assuming 4D input is NCHW\n",
      "  warnings.warn(\"Assuming 4D input is NCHW\")\n"
     ]
    }
   ],
   "source": [
    "from finn.util.pytorch import ToTensor\n",
    "from finn.transformation.merge_onnx_models import MergeONNXModels\n",
    "from finn.core.datatype import DataType\n",
    "\n",
    "#model = ModelWrapper(build_dir + \"/ckpt.t7.M5_11111_0920.pth.finn.onnx\")\n",
    "global_inp_name = model.graph.input[0].name\n",
    "ishape = model.get_tensor_shape(global_inp_name)\n",
    "# preprocessing: torchvision's ToTensor divides uint8 inputs by 255\n",
    "totensor_pyt = ToTensor()\n",
    "chkpt_preproc_name = build_dir+\"/ckpt.t7.M5_11111.pth.finn_preproc.onnx\"\n",
    "bo.export_finn_onnx(totensor_pyt, ishape, chkpt_preproc_name)\n",
    "\n",
    "# join preprocessing and core model\n",
    "pre_model = ModelWrapper(chkpt_preproc_name)\n",
    "model = model.transform(MergeONNXModels(pre_model))\n",
    "# add input quantization annotation: UINT8 for all BNN-PYNQ models\n",
    "global_inp_name = model.graph.input[0].name\n",
    "model.set_tensor_datatype(global_inp_name, DataType.UINT8)\n",
    "#model.set_tensor_datatype(global_inp_name, DataType.INT16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "43198899",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.finn_pre_post.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970e08280>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from finn.transformation.insert_topk import InsertTopK\n",
    "from finn.transformation.infer_datatypes import InferDataTypes\n",
    "from finn.core.datatype import DataType\n",
    "\n",
    "\n",
    "# postprocessing: insert Top-1 node at the end\n",
    "model = model.transform(InsertTopK(k=1))\n",
    "chkpt_name = build_dir+\"/ckpt.t7.M5_11111.pth.finn_pre_post.onnx\"\n",
    "# tidy-up again\n",
    "model = model.transform(InferShapes())\n",
    "model = model.transform(FoldConstants())\n",
    "model = model.transform(GiveUniqueNodeNames())\n",
    "model = model.transform(GiveReadableTensorNames())\n",
    "model = model.transform(InferDataTypes())\n",
    "model = model.transform(RemoveStaticGraphInputs())\n",
    "model.save(chkpt_name)\n",
    "\n",
    "showInNetron(build_dir+\"/ckpt.t7.M5_11111.pth.finn_pre_post.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "05a69573",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class Streamline(Transformation):\n",
      "    \"\"\"Apply the streamlining transform, see arXiv:1709.04060.\"\"\"\n",
      "\n",
      "    def apply(self, model):\n",
      "        streamline_transformations = [\n",
      "            ConvertSubToAdd(),\n",
      "            ConvertDivToMul(),\n",
      "            BatchNormToAffine(),\n",
      "            ConvertSignToThres(),\n",
      "            MoveMulPastMaxPool(),\n",
      "            MoveScalarLinearPastInvariants(),\n",
      "            AbsorbSignBiasIntoMultiThreshold(),\n",
      "            MoveAddPastMul(),\n",
      "            MoveScalarAddPastMatMul(),\n",
      "            MoveAddPastConv(),\n",
      "            MoveScalarMulPastMatMul(),\n",
      "            MoveScalarMulPastConv(),\n",
      "            MoveAddPastMul(),\n",
      "            CollapseRepeatedAdd(),\n",
      "            CollapseRepeatedMul(),\n",
      "            MoveMulPastMaxPool(),\n",
      "            AbsorbAddIntoMultiThreshold(),\n",
      "            FactorOutMulSignMagnitude(),\n",
      "            AbsorbMulIntoMultiThreshold(),\n",
      "            Absorb1BitMulIntoMatMul(),\n",
      "            Absorb1BitMulIntoConv(),\n",
      "            RoundAndClipThresholds(),\n",
      "        ]\n",
      "        for trn in streamline_transformations:\n",
      "            warnings.warn(str(type(trn)))\n",
      "            model = model.transform(trn)\n",
      "            model = model.transform(RemoveIdentityOps())\n",
      "            model = model.transform(GiveUniqueNodeNames())\n",
      "            model = model.transform(GiveReadableTensorNames())\n",
      "            model = model.transform(InferDataTypes())\n",
      "        return (model, False)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from finn.util.visualization import showSrc\n",
    "from finn.transformation.streamline import Streamline\n",
    "showSrc(Streamline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a59041a4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.finn_pre_stream.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970d88b80>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from finn.transformation.streamline.reorder import MoveScalarLinearPastInvariants\n",
    "import finn.transformation.streamline.absorb as absorb\n",
    "\n",
    "model = ModelWrapper(build_dir+\"/ckpt.t7.M5_11111.pth.finn_pre_post.onnx\")\n",
    "# move initial Mul (from preproc) past the Reshape\n",
    "model = model.transform(MoveScalarLinearPastInvariants())\n",
    "chkpt_name = build_dir+\"/ckpt.t7.M5_11111.pth.finn_pre_stream.onnx\"\n",
    "model.save(chkpt_name)\n",
    "#no use!!the picture is the same as last one \n",
    "showInNetron(build_dir+\"/ckpt.t7.M5_11111.pth.finn_pre_stream.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8ddc98e1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.general.ConvertSubToAdd'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.general.ConvertDivToMul'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.batchnorm_to_affine.BatchNormToAffine'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.sign_to_thres.ConvertSignToThres'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.reorder.MoveMulPastMaxPool'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.reorder.MoveScalarLinearPastInvariants'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.absorb.AbsorbSignBiasIntoMultiThreshold'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.reorder.MoveAddPastMul'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.reorder.MoveScalarAddPastMatMul'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.reorder.MoveAddPastConv'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.reorder.MoveScalarMulPastMatMul'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.reorder.MoveScalarMulPastConv'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.collapse_repeated.CollapseRepeatedAdd'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.collapse_repeated.CollapseRepeatedMul'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.absorb.AbsorbAddIntoMultiThreshold'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.absorb.FactorOutMulSignMagnitude'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.absorb.AbsorbMulIntoMultiThreshold'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.absorb.Absorb1BitMulIntoMatMul'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.absorb.Absorb1BitMulIntoConv'>\n",
      "  warnings.warn(str(type(trn)))\n",
      "/workspace/finn/src/finn/transformation/streamline/__init__.py:102: UserWarning: <class 'finn.transformation.streamline.round_thresholds.RoundAndClipThresholds'>\n",
      "  warnings.warn(str(type(trn)))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.finn_streamlined.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970da5340>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# streamline\n",
    "#model = model.transform(Streamline())\n",
    "model = model.transform(Streamline())\n",
    "model.save(build_dir+\"/ckpt.t7.M5_11111.pth.finn_streamlined.onnx\")\n",
    "showInNetron(build_dir+\"/ckpt.t7.M5_11111.pth.finn_streamlined.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "518d73bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from finn.transformation.streamline import Streamline\n",
    "from finn.transformation.lower_convs_to_matmul import LowerConvsToMatMul\n",
    "from finn.transformation.bipolar_to_xnor import ConvertBipolarMatMulToXnorPopcount\n",
    "import finn.transformation.streamline.absorb as absorb\n",
    "from finn.transformation.streamline.reorder import MakeMaxPoolNHWC, MoveScalarLinearPastInvariants\n",
    "from finn.transformation.infer_data_layouts import InferDataLayouts\n",
    "from finn.transformation.general import RemoveUnusedTensors\n",
    "\n",
    "\n",
    "model = ModelWrapper(build_dir+\"/ckpt.t7.M5_11111.pth.finn_streamlined.onnx\")\n",
    "model = model.transform(LowerConvsToMatMul())\n",
    "model = model.transform(MakeMaxPoolNHWC())\n",
    "model = model.transform(absorb.AbsorbTransposeIntoMultiThreshold())\n",
    "model = model.transform(ConvertBipolarMatMulToXnorPopcount())\n",
    "model.save(build_dir+\"/ckpt.t7.M5_11111.pth.finn_before_second_streamlined.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "583cfa15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.finn_before_second_streamlined.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970d95e80>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir+\"/ckpt.t7.M5_11111.pth.finn_before_second_streamlined.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7769916b",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = model.transform(Streamline())\n",
    "model.save(build_dir+\"/ckpt.t7.M5_11111.pth.finn_after_second_streamlined.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e150da80",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.finn_after_second_streamlined.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970da87c0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir+\"/ckpt.t7.M5_11111.pth.finn_after_second_streamlined.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e81540a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.transform(absorb.AbsorbScalarMulAddIntoTopK())\n",
    "model = model.transform(InferDataLayouts())\n",
    "model = model.transform(RemoveUnusedTensors())\n",
    "\n",
    "\n",
    "model.save(build_dir+\"/ckpt.t7.M5_11111.pth.finn_ready_for_hls_conversion.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fbfa336b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.finn_ready_for_hls_conversion.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970d88f70>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir+\"/ckpt.t7.M5_11111.pth.finn_ready_for_hls_conversion.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b4bcf522",
   "metadata": {},
   "outputs": [],
   "source": [
    "import finn.transformation.fpgadataflow.convert_to_hls_layers as to_hls\n",
    "from finn.transformation.fpgadataflow.create_dataflow_partition import (\n",
    "    CreateDataflowPartition,\n",
    ")\n",
    "\"\"\"\n",
    "from finn.transformation.move_reshape import RemoveCNVtoFCFlatten\n",
    "from finn.custom_op.registry import getCustomOp\n",
    "from finn.transformation.infer_data_layouts import InferDataLayouts\n",
    "from finn.transformation.fpgadataflow.make_zynq_proj import ZynqBuild\n",
    "from finn.core.modelwrapper import ModelWrapper\n",
    "from finn.custom_op.registry import getCustomOp\n",
    "from finn.util.basic import make_build_dir\n",
    "from finn.util.visualization import showInNetron\n",
    "import onnx\n",
    "from finn.util.test import get_test_model_trained\n",
    "import brevitas.onnx as bo\n",
    "from finn.core.modelwrapper import ModelWrapper\n",
    "from finn.transformation.infer_shapes import InferShapes\n",
    "from finn.transformation.fold_constants import FoldConstants\n",
    "from finn.transformation.general import GiveReadableTensorNames, GiveUniqueNodeNames, RemoveStaticGraphInputs\n",
    "from finn.transformation.streamline import Streamline\n",
    "from finn.transformation.lower_convs_to_matmul import LowerConvsToMatMul\n",
    "from finn.transformation.bipolar_to_xnor import ConvertBipolarMatMulToXnorPopcount\n",
    "import finn.transformation.streamline.absorb as absorb\n",
    "from finn.transformation.streamline.reorder import MakeMaxPoolNHWC, MoveScalarLinearPastInvariants\n",
    "from finn.transformation.infer_data_layouts import InferDataLayouts\n",
    "from finn.transformation.general import RemoveUnusedTensors\n",
    "build_dir = \"/workspace/finn\"\n",
    "\"\"\"\n",
    "# choose the memory mode for the MVTU units, decoupled or const\n",
    "mem_mode = \"decoupled\"\n",
    "\n",
    "model = ModelWrapper(build_dir + \"/ckpt.t7.M5_11111.pth.finn_ready_for_hls_conversion.onnx\")\n",
    "model = model.transform(to_hls.InferBinaryStreamingFCLayer(mem_mode))\n",
    "model = model.transform(to_hls.InferQuantizedStreamingFCLayer(mem_mode))\n",
    "# TopK to LabelSelect\n",
    "model = model.transform(to_hls.InferLabelSelectLayer())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a61c1e19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.finn_test.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970c68040>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# input quantization (if any) to standalone thresholding\n",
    "model = model.transform(to_hls.InferThresholdingLayer())\n",
    "model = model.transform(to_hls.InferConvInpGen())\n",
    "model.save(build_dir+\"/ckpt.t7.M5_11111.pth.finn_test.onnx\")\n",
    "showInNetron(build_dir+\"/ckpt.t7.M5_11111.pth.finn_test.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "52ea7c09",
   "metadata": {},
   "outputs": [],
   "source": [
    "from finn.transformation.move_reshape import RemoveCNVtoFCFlatten\n",
    "\n",
    "model = ModelWrapper(build_dir + \"/ckpt.t7.M5_11111.pth.finn_test.onnx\")\n",
    "model = model.transform(to_hls.InferStreamingMaxPool())\n",
    "# get rid of Reshape(-1, 1) operation between hlslib nodes\n",
    "model.save(build_dir + \"/ckpt.t7.M5_11111.pth.BRemoveCNVtoFCFlatten.onnx\")\n",
    "model = model.transform(RemoveCNVtoFCFlatten())\n",
    "model.save(build_dir + \"/ckpt.t7.M5_11111.pth.ARemoveCNVtoFCFlatten.onnx\")\n",
    "# get rid of Tranpose -> Tranpose identity seq\n",
    "model = model.transform(absorb.AbsorbConsecutiveTransposes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "52683a7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.BRemoveCNVtoFCFlatten.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970d95f10>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir + \"/ckpt.t7.M5_11111.pth.BRemoveCNVtoFCFlatten.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5ba0212a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from finn.custom_op.registry import getCustomOp\n",
    "# infer tensor data layouts\n",
    "model = model.transform(InferDataLayouts())\n",
    "parent_model = model.transform(CreateDataflowPartition())\n",
    "parent_model.save(build_dir + \"/ckpt.t7.M5_11111.pth.finn_dataflow_parent.onnx\")\n",
    "sdp_node = parent_model.get_nodes_by_op_type(\"StreamingDataflowPartition\")[0]\n",
    "sdp_node = getCustomOp(sdp_node)\n",
    "dataflow_model_filename = sdp_node.get_nodeattr(\"model\")\n",
    "# save the dataflow partition with a different name for easier access\n",
    "dataflow_model = ModelWrapper(dataflow_model_filename)\n",
    "dataflow_model.save(build_dir + \"/ckpt.t7.M5_11111.pth.finn_dataflow_model.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d795c136",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.finn_dataflow_model.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970d86220>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir + \"/ckpt.t7.M5_11111.pth.finn_dataflow_model.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3c33fa3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/0928/ckpt.t7.M5_11111.pth.finn_dataflow_parent.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970d861c0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir + \"/ckpt.t7.M5_11111.pth.finn_dataflow_parent.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e626d699",
   "metadata": {},
   "outputs": [],
   "source": [
    "#showInNetron(\"/tmp/finn_dev_lab2312/dataflow_partition0_ewnvlgpe/df_model.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "20ff207c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'from finn.custom_op.registry import getCustomOp\\nsdp_node = parent_model.get_nodes_by_op_type(\"StreamingDataflowPartition\")[0]\\nsdp_node = getCustomOp(sdp_node)\\ndataflow_model_filename = sdp_node.get_nodeattr(\"model\")\\nshowInNetron(dataflow_model_filename)'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''from finn.custom_op.registry import getCustomOp\n",
    "sdp_node = parent_model.get_nodes_by_op_type(\"StreamingDataflowPartition\")[0]\n",
    "sdp_node = getCustomOp(sdp_node)\n",
    "dataflow_model_filename = sdp_node.get_nodeattr(\"model\")\n",
    "showInNetron(dataflow_model_filename)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "93d7bde0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from finn.transformation.fpgadataflow.make_zynq_proj import ZynqBuild\n",
    "from finn.core.modelwrapper import ModelWrapper\n",
    "from finn.custom_op.registry import getCustomOp\n",
    "from finn.util.basic import make_build_dir\n",
    "from finn.util.visualization import showInNetron\n",
    "import onnx\n",
    "from finn.util.test import get_test_model_trained\n",
    "import brevitas.onnx as bo\n",
    "from finn.core.modelwrapper import ModelWrapper\n",
    "from finn.transformation.infer_shapes import InferShapes\n",
    "from finn.transformation.fold_constants import FoldConstants\n",
    "from finn.transformation.general import GiveReadableTensorNames, GiveUniqueNodeNames, RemoveStaticGraphInputs\n",
    "\n",
    "build_dir = \"/workspace/finn\"\n",
    "model = ModelWrapper(build_dir + \"/ckpt.t7.M5_11111.pth.finn_dataflow_model.onnx\")\n",
    "fc_layers = model.get_nodes_by_op_type(\"StreamingFCLayer_Batch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "fccbd4de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/ckpt.t7.M5_11111.pth.finn_dataflow_parent.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970c8b220>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir + \"/ckpt.t7.M5_11111.pth.finn_dataflow_parent.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3fc2d39c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/ckpt.t7.M5_11111.pth.finn_dataflow_model.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970c865e0>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir + \"/ckpt.t7.M5_11111.pth.finn_dataflow_model.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "97f27f27",
   "metadata": {},
   "outputs": [],
   "source": [
    "# each tuple is (PE, SIMD, in_fifo_depth, ramstyle) for a layer\n",
    "'''\n",
    "folding = [\n",
    "    (1, 1, 128, \"auto\"),\n",
    "    (1, 1, 128, \"auto\"),\n",
    "    (1, 1, 128, \"auto\"),\n",
    "    (1, 1, 128, \"auto\"),\n",
    "    (1, 1, 128, \"auto\"),\n",
    "    (1, 1, 128, \"auto\"),\n",
    "    (1, 1, 128, \"auto\"),\n",
    "    (1, 1, 128, \"auto\"),\n",
    "    (1, 1, 128, \"auto\"),\n",
    "    (1, 1, 3, \"auto\"),\n",
    "]\n",
    "'''\n",
    "folding = [\n",
    "    (8, 1, 128, \"distributed\"),\n",
    "    (8, 4, 32, \"distributed\"),\n",
    "    (8, 4, 32, \"auto\"),\n",
    "    (8, 4, 32, \"auto\"),\n",
    "    (8, 4, 32, \"block\"),\n",
    "    (8, 4, 32, \"auto\"),\n",
    "    (8, 4, 32, \"auto\"),\n",
    "    (8, 4, 128, \"auto\"),\n",
    "    (8, 4, 128, \"auto\"),\n",
    "    (1, 8, 3, \"block\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b1d2f5ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "URAM_counter = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "bf58f310",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''for fcl, (pe, simd, ififodepth, ramstyle) in zip(fc_layers, folding):\n",
    "    fcl_inst = getCustomOp(fcl)\n",
    "    \n",
    "    if URAM_counter < 5:\n",
    "        #fcl_inst.set_nodeattr(\"runtime_writeable_weights\", 1)\n",
    "        fcl_inst.set_nodeattr(\"mem_mode\", \"external\")\n",
    "    else:\n",
    "        fcl_inst.set_nodeattr(\"mem_mode\", \"decoupled\")\n",
    "        fcl_inst.set_nodeattr(\"ram_style\", ramstyle)\n",
    "    URAM_counter +=1\n",
    "    fcl_inst.set_nodeattr(\"PE\", pe)\n",
    "    fcl_inst.set_nodeattr(\"SIMD\", simd)\n",
    "    fcl_inst.set_nodeattr(\"inFIFODepth\", ififodepth)'''\n",
    "    #fcl_inst.set_nodeattr(\"mem_mode\", \"external\")\n",
    "    #fcl_inst.set_nodeattr(\"ram_style\", ramstyle)\n",
    "\n",
    "for fcl, (pe, simd, ififodepth, ramstyle) in zip(fc_layers, folding):\n",
    "    fcl_inst = getCustomOp(fcl)\n",
    "    '''if URAM_counter < 7:\n",
    "        fcl_inst.set_nodeattr(\"runtime_writeable_weights\", 1)\n",
    "        print(URAM_counter)\n",
    "    URAM_counter +=1'''\n",
    "    \n",
    "    fcl_inst.set_nodeattr(\"PE\", pe)\n",
    "    fcl_inst.set_nodeattr(\"SIMD\", simd)\n",
    "    fcl_inst.set_nodeattr(\"inFIFODepth\", ififodepth) \n",
    "    fcl_inst.set_nodeattr(\"ram_style\", ramstyle)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8ca6341f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "0\n",
      "9\n",
      "1\n",
      "9\n",
      "2\n",
      "9\n",
      "3\n",
      "9\n",
      "4\n",
      "9\n",
      "5\n",
      "9\n",
      "6\n",
      "9\n",
      "7\n",
      "9\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "URAM_counter = 0\n",
    "# use same SIMD values for the sliding window operators\n",
    "#swg_layers = model.get_nodes_by_op_type(\"ConvolutionInputGenerator\")\n",
    "swg_layers = model.get_nodes_by_op_type(\"ConvolutionInputGenerator1D\")\n",
    "for i in range(len(swg_layers)):\n",
    "    swg_inst = getCustomOp(swg_layers[i])\n",
    "    print(len(swg_layers))\n",
    "    simd = folding[i][1]\n",
    "    if URAM_counter < 1:\n",
    "        ram_style = \"ultra\"\n",
    "    else :\n",
    "        ram_style = \"distributed\"\n",
    "    print(URAM_counter)\n",
    "    URAM_counter +=1\n",
    "    \n",
    "    swg_inst.set_nodeattr(\"SIMD\", simd)\n",
    "    swg_inst.set_nodeattr(\"ram_style\", ram_style)\n",
    "\n",
    "model = model.transform(GiveUniqueNodeNames())\n",
    "model.save(build_dir + \"/ckpt.t7.M5_11111.pth.finn_folded.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "fb7af138",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/ckpt.t7.M5_11111.pth.finn_folded.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970d85190>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir + \"/ckpt.t7.M5_11111.pth.finn_folded.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "78d54c4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['Ultra96', 'Pynq-Z1', 'Pynq-Z2', 'ZCU102', 'ZCU104'])\n"
     ]
    }
   ],
   "source": [
    "# print the names of the supported PYNQ boards\n",
    "from finn.util.basic import pynq_part_map\n",
    "print(pynq_part_map.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "53f8b555",
   "metadata": {},
   "outputs": [],
   "source": [
    "# change this if you have a different PYNQ board, see list above\n",
    "pynq_board = \"ZCU104\"\n",
    "fpga_part = pynq_part_map[pynq_board]\n",
    "target_clk_ns = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "08970ab8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:105: UserWarning: get_folded_input_shape: 1, 1\n",
      "  warnings.warn(\"get_folded_input_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:133: UserWarning: get_folded_output_shape: 1, 1\n",
      "  warnings.warn(\"get_folded_output_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:105: UserWarning: get_folded_input_shape: 48, 4\n",
      "  warnings.warn(\"get_folded_input_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:133: UserWarning: get_folded_output_shape: 48, 4\n",
      "  warnings.warn(\"get_folded_output_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:105: UserWarning: get_folded_input_shape: 96, 4\n",
      "  warnings.warn(\"get_folded_input_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:133: UserWarning: get_folded_output_shape: 96, 4\n",
      "  warnings.warn(\"get_folded_output_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:105: UserWarning: get_folded_input_shape: 192, 4\n",
      "  warnings.warn(\"get_folded_input_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:133: UserWarning: get_folded_output_shape: 192, 4\n",
      "  warnings.warn(\"get_folded_output_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/transformation/fpgadataflow/floorplan.py:107: UserWarning: 37 nodes have no entry in the provided floorplan, SLR was set to -1\n",
      "  warnings.warn(\n",
      "/workspace/finn/src/finn/transformation/fpgadataflow/create_stitched_ip.py:90: UserWarning: The chosen frequency may lead to failure due to clock divider\n",
      "                constraints.\n",
      "  warnings.warn(\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:105: UserWarning: get_folded_input_shape: 1, 1\n",
      "  warnings.warn(\"get_folded_input_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:133: UserWarning: get_folded_output_shape: 1, 1\n",
      "  warnings.warn(\"get_folded_output_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/streamingdatawidthconverter_batch.py:86: UserWarning: inWidth: 8, outWidth: 4\n",
      "  warnings.warn(\"inWidth: %s, outWidth: %s\" %(str(iwidth), str(owidth)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:105: UserWarning: get_folded_input_shape: 48, 4\n",
      "  warnings.warn(\"get_folded_input_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:133: UserWarning: get_folded_output_shape: 48, 4\n",
      "  warnings.warn(\"get_folded_output_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/streamingdatawidthconverter_batch.py:86: UserWarning: inWidth: 8, outWidth: 48\n",
      "  warnings.warn(\"inWidth: %s, outWidth: %s\" %(str(iwidth), str(owidth)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/streamingdatawidthconverter_batch.py:86: UserWarning: inWidth: 48, outWidth: 4\n",
      "  warnings.warn(\"inWidth: %s, outWidth: %s\" %(str(iwidth), str(owidth)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:105: UserWarning: get_folded_input_shape: 96, 4\n",
      "  warnings.warn(\"get_folded_input_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:133: UserWarning: get_folded_output_shape: 96, 4\n",
      "  warnings.warn(\"get_folded_output_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/streamingdatawidthconverter_batch.py:86: UserWarning: inWidth: 8, outWidth: 96\n",
      "  warnings.warn(\"inWidth: %s, outWidth: %s\" %(str(iwidth), str(owidth)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/streamingdatawidthconverter_batch.py:86: UserWarning: inWidth: 96, outWidth: 4\n",
      "  warnings.warn(\"inWidth: %s, outWidth: %s\" %(str(iwidth), str(owidth)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:105: UserWarning: get_folded_input_shape: 192, 4\n",
      "  warnings.warn(\"get_folded_input_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/custom_op/fpgadataflow/convolutioninputgenerator1d.py:133: UserWarning: get_folded_output_shape: 192, 4\n",
      "  warnings.warn(\"get_folded_output_shape: %s, %s\" %(str(ifm_ch), str(simd)))\n",
      "/workspace/finn/src/finn/transformation/fpgadataflow/insert_fifo.py:154: UserWarning: Overriding input FIFO depth to 32\n",
      "  warnings.warn(\"Overriding input FIFO depth to 32\")\n",
      "/workspace/finn/src/finn/transformation/fpgadataflow/insert_fifo.py:200: UserWarning: Overriding output FIFO depth to 32\n",
      "  warnings.warn(\"Overriding output FIFO depth to 32\")\n",
      "/workspace/finn/src/finn/transformation/fpgadataflow/create_stitched_ip.py:90: UserWarning: The chosen frequency may lead to failure due to clock divider\n",
      "                constraints.\n",
      "  warnings.warn(\n",
      "/workspace/finn/src/finn/transformation/fpgadataflow/create_stitched_ip.py:90: UserWarning: The chosen frequency may lead to failure due to clock divider\n",
      "                constraints.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "model = ModelWrapper(build_dir+\"/ckpt.t7.M5_11111.pth.finn_folded.onnx\")\n",
    "model = model.transform(ZynqBuild(platform = pynq_board, period_ns = target_clk_ns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "dd6699ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(build_dir + \"/ckpt.t7.M5_11111.pth.finn_synth.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "36fca81a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping http://0.0.0.0:8081\n",
      "Serving '/workspace/finn/ckpt.t7.M5_11111.pth.finn_synth.onnx' at http://0.0.0.0:8081\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"400\"\n",
       "            src=\"http://localhost:8081/\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa970d86af0>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInNetron(build_dir + \"/ckpt.t7.M5_11111.pth.finn_synth.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c9cf0b40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome to PYNQ Linux, based on Ubuntu 18.04 (GNU/Linux 5.4.0-xilinx-v2020.1 aarch64)\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# set up the following values according to your own environment\n",
    "# FINN will use ssh to deploy and run the generated accelerator\n",
    "ip = os.getenv(\"PYNQ_IP\", \"192.168.0.3\")\n",
    "username = os.getenv(\"PYNQ_USERNAME\", \"xilinx\")\n",
    "password = os.getenv(\"PYNQ_PASSWORD\", \"xilinx\")\n",
    "port = os.getenv(\"PYNQ_PORT\", 22)\n",
    "target_dir = os.getenv(\"PYNQ_TARGET_DIR\", \"/home/xilinx/finn_dev_lab2312\")\n",
    "# set up ssh options to only allow publickey authentication\n",
    "options = \"-o PreferredAuthentications=publickey -o PasswordAuthentication=no\"\n",
    "\n",
    "# test access to PYNQ board\n",
    "! ssh {options} {username}@{ip} -p {port} cat /var/run/motd.dynamic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "ea61d10c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from finn.transformation.fpgadataflow.make_deployment import DeployToPYNQ\n",
    "\n",
    "model = model.transform(DeployToPYNQ(\"192.168.0.3\", \"22\", \"xilinx\", \"xilinx\", \"/home/xilinx/finn_dev_lab2312\"))\n",
    "model.save(build_dir + \"/ckpt.t7.M5_11111.pth.pynq_deploy.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "491eae4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/xilinx/finn_dev_lab2312/pynq_deployment_nqrj6c5k'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_dir_pynq = target_dir + \"/\" + model.get_metadata_prop(\"pynq_deployment_dir\").split(\"/\")[-1]\n",
    "target_dir_pynq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbd69507",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ModelWrapper(build_dir + \"/ckpt.t7.M5_11111.pth.pynq_deploy.onnx\")\n",
    "sdp_node_middle = getCustomOp(model.graph.node[1])\n",
    "postsynth_layers = sdp_node_middle.get_nodeattr(\"model\")\n",
    "\n",
    "#showInNetron(postsynth_layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f023a530",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ModelWrapper(postsynth_layers)\n",
    "model.model.metadata_props"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1081581",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ModelWrapper(build_dir + \"/ckpt.t7.M5_11111.pth.pynq_deploy.onnx\")\n",
    "model.model.metadata_props"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac7649b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "! ls {model.get_metadata_prop(\"vivado_pynq_proj\")}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31a3b369",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.model.metadata_props"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fc03e73",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir_pynq = '/home/xilinx/finn_dev_lab2312/pynq_deployment_kkl5g6nh'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61a25ca0",
   "metadata": {},
   "outputs": [],
   "source": [
    "! ssh -o PreferredAuthentications=publickey -o PasswordAuthentication=no xilinx@192.168.0.3 -p 22 'ls -l /home/xilinx/finn_dev_lab2312/pynq_deployment_kkl5g6nh'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a900b584",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pkg_resources as pk\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "fn = pk.resource_filename(\"finn.qnn-data\", \"KWS/up2.npy\")\n",
    "x = np.load(fn)\n",
    "x = x.reshape(1, 16000, 1)\n",
    "plt.imshow(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0efc3d18",
   "metadata": {},
   "outputs": [],
   "source": [
    "from finn.transformation.fpgadataflow.make_zynq_proj import ZynqBuild\n",
    "from finn.core.modelwrapper import ModelWrapper\n",
    "from finn.custom_op.registry import getCustomOp\n",
    "from finn.util.basic import make_build_dir\n",
    "from finn.util.visualization import showInNetron\n",
    "import onnx\n",
    "from finn.util.test import get_test_model_trained\n",
    "import brevitas.onnx as bo\n",
    "from finn.core.modelwrapper import ModelWrapper\n",
    "from finn.transformation.infer_shapes import InferShapes\n",
    "from finn.transformation.fold_constants import FoldConstants\n",
    "from finn.transformation.general import GiveReadableTensorNames, GiveUniqueNodeNames, RemoveStaticGraphInputs\n",
    "from finn.util.visualization import showInNetron\n",
    "\n",
    "import numpy as np\n",
    "from finn.core.onnx_exec import execute_onnx\n",
    "\n",
    "\n",
    "build_dir = \"/workspace/finn\"\n",
    "model = ModelWrapper(build_dir + \"/ckpt.t7.M5_11111.pth.pynq_deployonboard.onnx\")\n",
    "iname = model.graph.input[0].name\n",
    "oname = model.graph.output[0].name\n",
    "ishape = model.get_tensor_shape(iname)\n",
    "print(iname)\n",
    "print(\"Expected network input shape is \" + str(ishape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "759b49aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "showInNetron(build_dir + \"/ckpt.t7.M5_11111.pth.pynq_deployonboard.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4051b3ce",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "input_dict = {iname: x.reshape(ishape)}\n",
    "input_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99b3f8e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "ret = execute_onnx(model, input_dict, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95c7a79e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ret[oname]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acc6edbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "! ssh -o PreferredAuthentications=publickey -o PasswordAuthentication=no xilinx@192.168.0.3 -p 22 'echo xilinx | sudo -S pip3 install git+https://github.com/fbcotter/dataset_loading.git@0.0.4#egg=dataset_loading'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "315f45a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from finn.core.throughput_test import throughput_test_remote\n",
    "\n",
    "model = ModelWrapper(build_dir + \"/ckpt.t7.M5_11111.pth.pynq_deployonboard.onnx\")\n",
    "res = throughput_test_remote(model, 10000)\n",
    "print(\"Network metrics:\")\n",
    "for key in res:\n",
    "    print(str(key) + \": \" + str(res[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c44d6a6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}