example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from bnn_trust_regions.candidate_region_identification import IdentGifSettings, SisoCandidateRegionIdentification\n",
    "from bnn_trust_regions.gaussian import UnivariateGaussian\n",
    "from bnn_trust_regions.io_data import IOData\n",
    "from bnn_trust_regions.model_evaluator import ModelEvaluator, UseAorB\n",
    "from bnn_trust_regions.plot_candidate_regions import ErrorbarPlotSettings, PlotSettings, DistributionPlotSettings\n",
    "from bnn_trust_regions.stat_test_settings import StatTestSettings\n",
    "from bnn_trust_regions.utils import save_load, matplotlib_settings\n",
    "from bnn_trust_regions.wasserstein_dist import WassersteinDistance\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Default settings for matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[6;30;42mUsing LaTeX for typesetting in matplotlib.\u001b[0m\n",
      "\u001b[1;37;41mMake sure to have LaTeX installed on your system or set use_tex=False to use matplotlib's default typesetting.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "matplotlib_settings.init_settings(use_tex=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load example Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################################################################################################\n",
    "# load test data\n",
    "########################################################################################################\n",
    "# folder with some example data and predictions\n",
    "data_folder = \"example_data\"\n",
    "# load test data\n",
    "test_data_file_name = \"nn_test\"\n",
    "test_input, test_output = save_load.load_io_data(data_folder, test_data_file_name)\n",
    "test_data = IOData(input=test_input, output=test_output)\n",
    "\n",
    "# load training data\n",
    "train_data_file_name = \"nn_train\"\n",
    "train_input, train_output = save_load.load_io_data(data_folder, train_data_file_name)\n",
    "train_data = IOData(input=train_input, output=train_output)\n",
    "\n",
    "# load predictions\n",
    "mcmc_file_name = \"mcmc_test\"\n",
    "pred_mcmc = save_load.load_sampled_predictions(data_folder, mcmc_file_name)\n",
    "svi_file_name = \"meanfield_svi_test\"\n",
    "pred_svi = save_load.load_sampled_predictions(data_folder, svi_file_name)\n",
    "pbp_file_name = \"pbp_test\"\n",
    "pred_pbp_mean, pred_pbp_var = save_load.load_sampled_predictions(data_folder, pbp_file_name)\n",
    "pred_pbp = UnivariateGaussian(mean=pred_pbp_mean, var=pred_pbp_var)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Settings and Hyperparameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################################################################################################\n",
    "#  change predictions and test data and hyperparameters\n",
    "# for region identification and statistical testing here\n",
    "########################################################################################################\n",
    "\n",
    "# set wasserstein distance settings\n",
    "p_norm = 1\n",
    "parallel_computing = True\n",
    "verbose = False\n",
    "ws_dist_settings1 = WassersteinDistance(\n",
    "    p_norm=p_norm, parallel_computing=parallel_computing, verbose=verbose)\n",
    "\n",
    "\n",
    "# candidate region identification settings\n",
    "min_points_per_region = 200\n",
    "smoothing_window_size = 50\n",
    "plot_gif = True\n",
    "\n",
    "# display gif of regions identification critical distance\n",
    "plot_folder1 = \"eval1_plots\"\n",
    "plot_folder2 = \"eval2_plots\"\n",
    "file_name = \"crit_dist\"\n",
    "dpi = 200\n",
    "fps = 2\n",
    "loop = 0  # 0 for infinite loop\n",
    "gif_settings1 = IdentGifSettings(\n",
    "    path=plot_folder1, file_name=file_name, dpi=dpi, fps=fps, loop=loop)\n",
    "\n",
    "\n",
    "region_ident1 = SisoCandidateRegionIdentification(\n",
    "    min_points_per_region=min_points_per_region, smoothing_window_size=smoothing_window_size, verbose=plot_gif, gif_settings=gif_settings1)\n",
    "\n",
    "\n",
    "# statistical test settings\n",
    "alpha = 0.01  # significance level of 1%\n",
    "confidence_interval = 0.95  # test the 95% confidence interval\n",
    "stat_test_settings = StatTestSettings(alpha=alpha, confidence_interval=confidence_interval)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "error_bar_plot_settings = ErrorbarPlotSettings(\n",
    "        # label\n",
    "        anees_label=r'ANEES is $\\chi^2$',\n",
    "        anees_label_notchi2=r'ANEES is not $\\chi^2$',\n",
    "        annes_errorbar_label='ANEES Bounds',\n",
    "        split_label='Region Split',\n",
    "        out_of_scope_label='ANEES o.s.',\n",
    "        anees_y_label='ANEES',\n",
    "        # colors\n",
    "        anees_bar_color='tab:orange',\n",
    "        anees_marker_color='tab:blue',\n",
    "        binom_bar_color='tab:purple',\n",
    "        binom_marker_color='tab:red',\n",
    "    )\n",
    "plot_settings = PlotSettings(\n",
    "    image_format='svg',  # image format of the plots\n",
    "    # plot_folder='eval1_plots',  # folder where the plots are saved\n",
    "    confidence_interval=0.95,\n",
    "    # settings to plot the predictions and the mean of the ground truth\n",
    "    ground_truth_plot_settings=DistributionPlotSettings(\n",
    "        mean_color='tab:orange',\n",
    "        mean_linestyle='-',\n",
    "        mean_label=r'$y=x^3$',),  # label for mean value of ground truth\n",
    "    wasserstein_plot_settings=DistributionPlotSettings(\n",
    "        mean_color='k',\n",
    "        mean_linestyle='-',\n",
    "        mean_label=r'$W_1^\\text{GT}$',),  # label for wasserstein distance between ground truth distribution and predictive distribution\n",
    "    error_bar_plot_settings=error_bar_plot_settings,\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "models to be tested"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pred_a = pred_mcmc # reference model\n",
    "pred_b = [pred_svi, pred_pbp] # list of models to compare with reference model\n",
    "plot_folder = ['eval1_plots', 'eval2_plots'] # list of folders where the plots are saved\n",
    "model_a_names = 'MCMC' # name of reference model\n",
    "model_b_names = ['SVI', 'PBP'] # list of model names\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ground truth (only for plotting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ground_truth_mean = np.power(test_data.input, 3).reshape(1, -1)\n",
    "ground_truth_distribution = UnivariateGaussian(mean=ground_truth_mean.squeeze(), var=9.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model MCMC:\n",
      "Binomial test results:\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "| x_min | x_max | proportion inside | p-value | prop CI low | prop CI high | calibrated predictions |\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "| -6.00 | -4.89 |        0.91       |   0.01  |     0.85    |     0.95     |          True          |\n",
      "| -4.89 | -3.77 |        0.92       |   0.03  |     0.86    |     0.96     |          True          |\n",
      "| -3.77 | -2.66 |        0.91       |   0.03  |     0.86    |     0.96     |          True          |\n",
      "| -2.66 | -1.54 |        0.91       |   0.01  |     0.84    |     0.95     |         False          |\n",
      "| -1.54 | -0.54 |        1.00       |   0.00  |     0.97    |     1.00     |         False          |\n",
      "| -0.54 |  0.47 |        1.00       |   0.00  |     0.97    |     1.00     |         False          |\n",
      "|  0.47 |  1.47 |        1.00       |   0.00  |     0.97    |     1.00     |         False          |\n",
      "|  1.47 |  2.76 |        0.95       |   0.89  |     0.90    |     0.98     |          True          |\n",
      "|  2.76 |  4.05 |        0.97       |   0.32  |     0.92    |     0.99     |          True          |\n",
      "|  4.05 |  5.34 |        0.95       |   0.67  |     0.90    |     0.98     |          True          |\n",
      "|  5.34 |  6.00 |        0.36       |   0.00  |     0.26    |     0.48     |         False          |\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "Model MCMC:\n",
      "ANEES test results:\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "| x_min | x_max | anees | p-value | anees crit bound low | anees crit bound high | calibrated predictions | nees is chi2 |\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "| -6.00 | -4.89 |  1.66 |   0.00  |         0.77         |          1.26         |         False          |    False     |\n",
      "| -4.89 | -3.77 |  1.18 |   0.06  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -3.77 | -2.66 |  1.16 |   0.11  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -2.66 | -1.54 |  1.16 |   0.09  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -1.54 | -0.54 |  0.18 |   0.00  |         0.76         |          1.28         |         False          |    False     |\n",
      "| -0.54 |  0.47 |  0.04 |   0.00  |         0.76         |          1.28         |         False          |    False     |\n",
      "|  0.47 |  1.47 |  0.16 |   0.00  |         0.76         |          1.28         |         False          |    False     |\n",
      "|  1.47 |  2.76 |  0.97 |   0.78  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  2.76 |  4.05 |  0.90 |   0.27  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  4.05 |  5.34 |  0.96 |   0.65  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  5.34 |  6.00 |  6.02 |   0.00  |         0.71         |          1.34         |         False          |    False     |\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "Model SVI:\n",
      "Binomial test results:\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "| x_min | x_max | proportion inside | p-value | prop CI low | prop CI high | calibrated predictions |\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "| -6.00 | -4.89 |        0.52       |   0.00  |     0.44    |     0.61     |         False          |\n",
      "| -4.89 | -3.77 |        0.93       |   0.22  |     0.88    |     0.97     |          True          |\n",
      "| -3.77 | -2.66 |        0.91       |   0.03  |     0.86    |     0.96     |          True          |\n",
      "| -2.66 | -1.54 |        0.91       |   0.03  |     0.86    |     0.96     |          True          |\n",
      "| -1.54 | -0.54 |        0.96       |   0.87  |     0.90    |     0.98     |          True          |\n",
      "| -0.54 |  0.47 |        0.94       |   0.51  |     0.88    |     0.98     |          True          |\n",
      "|  0.47 |  1.47 |        0.95       |   0.74  |     0.89    |     0.98     |          True          |\n",
      "|  1.47 |  2.76 |        0.94       |   0.57  |     0.89    |     0.97     |          True          |\n",
      "|  2.76 |  4.05 |        0.96       |   0.77  |     0.91    |     0.98     |          True          |\n",
      "|  4.05 |  5.34 |        0.95       |   0.89  |     0.90    |     0.98     |          True          |\n",
      "|  5.34 |  6.00 |        0.15       |   0.00  |     0.08    |     0.25     |         False          |\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "Model SVI:\n",
      "ANEES test results:\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "| x_min | x_max | anees | p-value | anees crit bound low | anees crit bound high | calibrated predictions | nees is chi2 |\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "| -6.00 | -4.89 |  6.48 |   0.00  |         0.77         |          1.26         |         False          |    False     |\n",
      "| -4.89 | -3.77 |  1.07 |   0.43  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -3.77 | -2.66 |  1.11 |   0.26  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -2.66 | -1.54 |  1.14 |   0.15  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -1.54 | -0.54 |  0.99 |   0.92  |         0.76         |          1.28         |          True          |     True     |\n",
      "| -0.54 |  0.47 |  1.16 |   0.12  |         0.76         |          1.28         |          True          |     True     |\n",
      "|  0.47 |  1.47 |  0.93 |   0.48  |         0.76         |          1.28         |          True          |     True     |\n",
      "|  1.47 |  2.76 |  0.97 |   0.77  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  2.76 |  4.05 |  0.89 |   0.19  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  4.05 |  5.34 |  0.93 |   0.42  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  5.34 |  6.00 | 17.38 |   0.00  |         0.71         |          1.34         |         False          |    False     |\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "Model MCMC:\n",
      "Binomial test results:\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "| x_min | x_max | proportion inside | p-value | prop CI low | prop CI high | calibrated predictions |\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "| -6.00 | -4.89 |        0.91       |   0.01  |     0.85    |     0.95     |          True          |\n",
      "| -4.89 | -3.77 |        0.92       |   0.03  |     0.86    |     0.96     |          True          |\n",
      "| -3.77 | -2.66 |        0.91       |   0.03  |     0.86    |     0.96     |          True          |\n",
      "| -2.66 | -1.54 |        0.91       |   0.01  |     0.84    |     0.95     |         False          |\n",
      "| -1.54 | -0.54 |        1.00       |   0.00  |     0.97    |     1.00     |         False          |\n",
      "| -0.54 |  0.47 |        1.00       |   0.00  |     0.97    |     1.00     |         False          |\n",
      "|  0.47 |  1.47 |        1.00       |   0.00  |     0.97    |     1.00     |         False          |\n",
      "|  1.47 |  2.76 |        0.95       |   0.89  |     0.90    |     0.98     |          True          |\n",
      "|  2.76 |  4.05 |        0.97       |   0.32  |     0.92    |     0.99     |          True          |\n",
      "|  4.05 |  5.34 |        0.95       |   0.67  |     0.90    |     0.98     |          True          |\n",
      "|  5.34 |  6.00 |        0.36       |   0.00  |     0.26    |     0.48     |         False          |\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "Model MCMC:\n",
      "ANEES test results:\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "| x_min | x_max | anees | p-value | anees crit bound low | anees crit bound high | calibrated predictions | nees is chi2 |\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "| -6.00 | -4.89 |  1.66 |   0.00  |         0.77         |          1.26         |         False          |    False     |\n",
      "| -4.89 | -3.77 |  1.18 |   0.06  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -3.77 | -2.66 |  1.16 |   0.11  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -2.66 | -1.54 |  1.16 |   0.09  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -1.54 | -0.54 |  0.18 |   0.00  |         0.76         |          1.28         |         False          |    False     |\n",
      "| -0.54 |  0.47 |  0.04 |   0.00  |         0.76         |          1.28         |         False          |    False     |\n",
      "|  0.47 |  1.47 |  0.16 |   0.00  |         0.76         |          1.28         |         False          |    False     |\n",
      "|  1.47 |  2.76 |  0.97 |   0.78  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  2.76 |  4.05 |  0.90 |   0.27  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  4.05 |  5.34 |  0.96 |   0.65  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  5.34 |  6.00 |  6.02 |   0.00  |         0.71         |          1.34         |         False          |    False     |\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "Model PBP:\n",
      "Binomial test results:\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "| x_min | x_max | proportion inside | p-value | prop CI low | prop CI high | calibrated predictions |\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "| -6.00 | -4.89 |        0.36       |   0.00  |     0.28    |     0.45     |         False          |\n",
      "| -4.89 | -3.77 |        0.95       |   1.00  |     0.90    |     0.98     |          True          |\n",
      "| -3.77 | -2.66 |        0.94       |   0.36  |     0.88    |     0.97     |          True          |\n",
      "| -2.66 | -1.54 |        0.94       |   0.36  |     0.88    |     0.97     |          True          |\n",
      "| -1.54 | -0.54 |        0.97       |   0.25  |     0.92    |     0.99     |          True          |\n",
      "| -0.54 |  0.47 |        0.96       |   0.87  |     0.90    |     0.98     |          True          |\n",
      "|  0.47 |  1.47 |        0.97       |   0.25  |     0.92    |     0.99     |          True          |\n",
      "|  1.47 |  2.76 |        0.96       |   0.48  |     0.92    |     0.99     |          True          |\n",
      "|  2.76 |  4.05 |        0.97       |   0.20  |     0.93    |     0.99     |          True          |\n",
      "|  4.05 |  5.34 |        0.95       |   0.89  |     0.90    |     0.98     |          True          |\n",
      "|  5.34 |  6.00 |        0.12       |   0.00  |     0.06    |     0.21     |         False          |\n",
      "+-------+-------+-------------------+---------+-------------+--------------+------------------------+\n",
      "Model PBP:\n",
      "ANEES test results:\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "| x_min | x_max | anees | p-value | anees crit bound low | anees crit bound high | calibrated predictions | nees is chi2 |\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n",
      "| -6.00 | -4.89 | 17.30 |   0.00  |         0.77         |          1.26         |         False          |    False     |\n",
      "| -4.89 | -3.77 |  1.03 |   0.69  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -3.77 | -2.66 |  1.03 |   0.72  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -2.66 | -1.54 |  1.06 |   0.53  |         0.77         |          1.26         |          True          |     True     |\n",
      "| -1.54 | -0.54 |  0.80 |   0.04  |         0.76         |          1.28         |          True          |     True     |\n",
      "| -0.54 |  0.47 |  1.07 |   0.45  |         0.76         |          1.28         |          True          |     True     |\n",
      "|  0.47 |  1.47 |  0.84 |   0.09  |         0.76         |          1.28         |          True          |     True     |\n",
      "|  1.47 |  2.76 |  0.90 |   0.27  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  2.76 |  4.05 |  0.82 |   0.03  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  4.05 |  5.34 |  1.00 |   0.96  |         0.79         |          1.24         |          True          |     True     |\n",
      "|  5.34 |  6.00 | 25.15 |   0.00  |         0.71         |          1.34         |         False          |    False     |\n",
      "+-------+-------+-------+---------+----------------------+-----------------------+------------------------+--------------+\n"
     ]
    }
   ],
   "source": [
    "for idx, preds in enumerate(pred_b):\n",
    "    # use different plot folder for each model\n",
    "    plot_settings.plot_folder = plot_folder[idx]\n",
    "\n",
    "    # set model names \n",
    "    model_names=(model_a_names, model_b_names[idx])\n",
    "\n",
    "    # set the models and get the wasserstein distance between the predictions\n",
    "    model_evaluator1 = ModelEvaluator(\n",
    "        predictions_a=pred_a, predictions_b=preds,\n",
    "        wasserstein_distance=ws_dist_settings1, test_data=test_data)\n",
    "    model_evaluator1.calc_wasserstein_distance()\n",
    "\n",
    "    # calculate candidate regions and plot gif of critical distance\n",
    "    model_evaluator1.calc_canidate_regions(region_ident=region_ident1)\n",
    "\n",
    "    # calculate statistical tests (ANEES and binomial test)\n",
    "    model_evaluator1.calc_statistical_tests(\n",
    "        stat_test_settings=stat_test_settings, )\n",
    "    \n",
    "    # print results to console\n",
    "    model_evaluator1.print_statistical_tests(model_names=model_names)  \n",
    "\n",
    "    # plot results to files\n",
    "    model_evaluator1.plot_statistical_tests(\n",
    "        plot_settings=plot_settings,\n",
    "          ground_truth=ground_truth_distribution, \n",
    "          model_names=model_names)\n",
    "\n",
    "    \n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "bnnTrustRegions",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}