docs/usage.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "from datetime import timedelta\n",
    "\n",
    "sys.path.append(os.path.abspath('../prognostics_benchmark'))\n",
    "from prognostics_benchmark import Benchmark\n",
    "from prognostics_benchmark.detectors import XGBClassifierDetector, MultiModelHTMDetector\n",
    "from prognostics_benchmark.evaluators import MaintenanceCostEvaluator"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "This notebook contains examples on how to use the benchmark as a module.\n",
    "\n",
    "### Basic Usage"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "benchmark = Benchmark()\n",
    "\n",
    "# get information on current content\n",
    "print(benchmark.dataset_ids)\n",
    "print(benchmark.get_detector_names())\n",
    "print(benchmark.get_evaluator_names())\n",
    "\n",
    "# evaluate an algorithm on the benchmark\n",
    "res = benchmark.score(\n",
    "        detector_name='MultiModelHTMDetector',\n",
    "        evaluator_name='MaintenanceCostEvaluator',\n",
    "        verbose=True\n",
    ")\n",
    "\n",
    "# you can also overwrite the default params and the default config of the detector and the evaluator\n",
    "my_evaluator_config = {\n",
    "    'harddrive': {\n",
    "        'lead_time': timedelta(days=4),\n",
    "        'relevant_maintenance_factor': 5,\n",
    "        'cost_rate': 20,\n",
    "    },\n",
    "    'turbofan_engine': {\n",
    "        'lead_time': timedelta(days=5),\n",
    "        'relevant_maintenance_factor': 3,\n",
    "        'cost_rate': 50.33,\n",
    "    },\n",
    "    'water_pump': {\n",
    "        'lead_time': timedelta(hours=36),\n",
    "        'relevant_maintenance_factor': 3,\n",
    "        'cost_rate': 7.01,\n",
    "    },\n",
    "    'production_plant': {\n",
    "        'lead_time': timedelta(hours=2),\n",
    "        'relevant_maintenance_factor': 5,\n",
    "        'cost_rate': 18.13,\n",
    "    }\n",
    "}\n",
    "res = benchmark.score(\n",
    "        detector_name='MultiModelHTMDetector',\n",
    "        evaluator_name='MaintenanceCostEvaluator',\n",
    "        evaluator_config=my_evaluator_config,\n",
    "        verbose=True\n",
    ")\n",
    "# you can also specify dataset ids to only evaluate on certain datasets\n",
    "res = benchmark.score(\n",
    "        detector_name='MultiModelHTMDetector',\n",
    "        evaluator_name='MaintenanceCostEvaluator',\n",
    "        dataset_ids=['turbofan_engine', 'harddrive'],\n",
    "        verbose=True\n",
    ")\n",
    "\n",
    "# do stuff directly on a dataset\n",
    "turbofan_engine_dataset = benchmark.get_dataset('turbofan_engine')\n",
    "print(turbofan_engine_dataset.model_ids)\n",
    "print(turbofan_engine_dataset.get_equipment_ids_for_model('FD001'))\n",
    "\n",
    "print(turbofan_engine_dataset.rtf_ids)\n",
    "print(turbofan_engine_dataset.get_rtf_ids_for_model(model_id='FD001'))\n",
    "print(turbofan_engine_dataset.get_rtf_ids_for_equipment(model_id='FD001', equipment_id='1'))\n",
    "\n",
    "turbofan_engine_evaluator = MaintenanceCostEvaluator(dataset_id='turbofan_engine')\n",
    "\n",
    "# This will create one instance of Detector for each model\n",
    "res = turbofan_engine_dataset.evaluate_on_models(\n",
    "    Detector=MultiModelHTMDetector,\n",
    "    evaluator=turbofan_engine_evaluator\n",
    ")\n",
    "# You can also specify specific models\n",
    "res = turbofan_engine_dataset.evaluate_on_models(\n",
    "    Detector=MultiModelHTMDetector,\n",
    "    evaluator=turbofan_engine_evaluator,\n",
    "    model_ids=['FD001', 'FD002']\n",
    ")\n",
    "\n",
    "# This will create one instance of Detector for each equipment\n",
    "res = turbofan_engine_dataset.evaluate_on_equipments(\n",
    "    Detector=MultiModelHTMDetector,\n",
    "    evaluator=turbofan_engine_evaluator,\n",
    "    model_id='FD001'\n",
    ")\n",
    "# Again, you can specify equipments\n",
    "res = turbofan_engine_dataset.evaluate_on_equipments(\n",
    "    Detector=MultiModelHTMDetector,\n",
    "    evaluator=turbofan_engine_evaluator,\n",
    "    model_id='FD001',\n",
    "    equipment_ids=['1', '2', '3']\n",
    ")\n",
    "\n",
    "# do stuff directly on a RtF\n",
    "rtf = turbofan_engine_dataset.get_rtf('FD001_1_0')\n",
    "print(rtf.id)\n",
    "print(rtf.equi_id)\n",
    "print(rtf.model_id)\n",
    "print(rtf.dataset_id)\n",
    "\n",
    "# Get time series\n",
    "df_rtf = rtf.get_df()\n",
    "\n",
    "# Evaluate on a RtF\n",
    "detector = MultiModelHTMDetector(evaluator=turbofan_engine_evaluator)\n",
    "df_results = rtf.score(detector=detector, verbose=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Additional Functionalities\n",
    "#### Parameter Optimization\n",
    "You can use the optimizer to optimize the parameters of your algorithm. It is based on [Bayesian Optimization](https://github.com/fmfn/BayesianOptimization) and allows you to run parameter optimization on the entire corpus, specific datasets, models, equipments and even individual RtFs."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from prognostics_benchmark.optimization.optimizer import Optimizer\n",
    "\n",
    "# Provide bounds for the parameters you want to optimize.\n",
    "# For more information read about bayesian optimization.\n",
    "p_bounds = {\n",
    "    'short_htm_probationary_period': (20, 100),\n",
    "    'short_spot_probationary_period_ratio': (0.1, 1),\n",
    "    'short_smoothing_kernel_size_ratio': (0.1, 1),\n",
    "    'short_spot_q': (0.0001, 0.5),\n",
    "    'short_spot_depth_ratio': (0.1, 0.6),\n",
    "    'long_htm_probationary_period': (106, 5000),\n",
    "    'long_spot_probationary_period_ratio': (0.1, 1),\n",
    "    'long_smoothing_kernel_size_ratio': (0.1, 1),\n",
    "    'long_spot_q': (0.0001, 0.5),\n",
    "    'long_spot_depth_ratio': (0.1, 0.6),\n",
    "}\n",
    "\n",
    "optimizer = Optimizer(\n",
    "    out_path=os.path.join('opt_logs'),  # path to which to write the logs\n",
    "    p_bounds=p_bounds,  # bounds dict\n",
    "    # if true, it checks if bounds are provided for all parameters and\n",
    "    # raises an exception otherwise.\n",
    "    # If false, it fills up missing parameters with their default setting.\n",
    "    exact=True,\n",
    "    evaluator_name='MaintenanceCostEvaluator',\n",
    "    detector_name='MultiModelHTMDetector',\n",
    "    init_points = 5,\n",
    "    n_iter = 15\n",
    ")\n",
    "optimizer.optimize()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "You can also provide dataset_id, dataset_ids, model_ids and rtf_id depending on on what you want to optimize.\n",
    "+ If rtf_id is provided, it will optimize on that RtF and ignore all other parameters\n",
    "+ If model_ids is provided, it will optimize on these models. Requires dataset_id to be also set.\n",
    "+ If dataset_id is set but not model_ids, it optimizes on all models of the given dataset\n",
    "+ If none of the above is set, it will optimize on the entire benchmark.\n",
    "#### Plotting\n",
    "Both, evaluator and detector, have plotting functionalities which can be overwritten by the respective implementation."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [

    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'evaluation': 92.808, 'failure_prevented': True, 'n_maintenance_activities': 8, 'plot': <Figure size 432x288 with 1 Axes>}\n"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAABL80lEQVR4nO2daXhURdaA3+rOnhBCEghhD8qeDQibbFF2FWRVEIXIAAIi4jc6g8soMjCD4oyKGwMj4IIIwgCuIwbIsChgQPbFgAQIa0ggZE+6u74ft9N0yE6Wzk3qfZ773O6qunVO3U7OrXuq6pSQUqJQKBSK2oXB0QooFAqFoupRxl+hUChqIcr4KxQKRS1EGX+FQqGohSjjr1AoFLUQJ0crUFr8/f1lixYtHK2GQqFQFMnJk9q5TZvqIWzfvn3XpJT1C8vTjfFv0aIFsbGxjlZDoVAoiiQyUjvHxFQPYUKIs0XlKbePQqFQ1EJ00/NXKBSK6s7LL+tHmDL+CoVCUUH0768fYcr4K2o1ubm5JCQkkJWV5WhVFDWAnBzt7OJStcLc3Nxo0qQJzs7Opb5cGX9FrSYhIYE6derQokULhBCOVkehcxwx20e2bk1SUhIJCQkEBQWV+nI14Kuo1WRlZeHn56cMv0K3CCHw8/Mr89urMv6KWo8y/Aq9cyd/w8r4KxQKRS1EGX+FoqzcPKkdFcjGjRsRQnDixIkKrbeseHl5lbqsxWJh1qxZBAcHExISQpcuXThz5kzZBJZ0L8uTf6e/U2XUWUpOXjvJyWuVV789asBXoagGrF69ml69erF69Wpee+01R6tTKtasWcPFixc5dOgQBoOBhIQEPD09y1WnyWTCyUm/ZqlxY/0IUz1/hcLBpKWlsXPnTj766CO++OILW3pMTAyRkZGMHj2atm3bMn78ePJ23tuyZQsdO3YkJCSESZMmkZ2dDWhhUF544QXCw8OJiIhg//79DBo0iLvuuoslS5bY5PXr149OnToREhLCpk2bCug0YcIENm7caPs+fvz4AuUuXbpEYGAgBoNmRpo0aUK9evUA+O9//0unTp0ICwujX79+ACQnJzN8+HBCQ0Pp3r07hw4dAmDu39/l8ccfp2fPnjz++OMkJiYyatQounTpQpfI0ezavb8ibnOV4OWlHXoQpt9HrEJRwXy2dT/nrt4ouaApQzs7XSixaLMGPjx2X6diy2zatInBgwfTunVr/Pz82LdvH507dwbg119/5ejRozRq1IiePXuya9cuIiIiiIqKYsuWLbRu3ZoJEybw4YcfMnv2bE1ms2YcOHCAZ599lqioKHbt2kVWVhbBwcFMmzYNNzc3NmzYgLe3N9euXaN79+4MGzYs36DhH/7wB9566y2GDx9OSkoKP/30Ex9//HE+vR9++GF69erFjh076NevH4899hgdO3YkMTGRKVOmsH37doKCgkhOTgbg1VdfpWPHjmzcuJGtW7cyYcIEDmxfA8CxY8fYuXMn7u7uPProozz77LP06tWLc0e3MWjkZI6fHFfy71INSEvTzlXyACinMGX8FQoHs3r1ap555hkAxo4dy+rVq23Gv2vXrjRp0gSA8PBw4uPjqVOnDkFBQbRu3RqAiRMn8v7779uM/7BhwwAICQkhLS2NOnXqUKdOHVxdXblx4waenp68+OKLbN++HYPBwIULF7hy5QoNGza06dS3b19mzJhBYmIi69evZ9SoUQXcMU2aNOHkyZNs3bqVrVu30q9fP7788ksyMjLo06ePbc65r68vADt37mT9+vUA3HfffSQlJXHzZppNZ3d3dwCio6M5duyYJsSSzc3UNNLS0so0HuEoLlj7A1Uyz7+cwpTxVyislNRDt5E34Odd/v/w5ORktm7dyuHDhxFCYDabEUKwaNEiAFxdXW1ljUYjJpOpxDrzrjEYDPmuNxgMmEwmVq1aRWJiIvv27cPZ2ZkWLVoUOkd8woQJfPbZZ3zxxResWLGiSFlDhgxhyJAhBAQEsHHjRgYOHFimewDkGyuwWCzs3r0bNze3W/daB4ZfbzjM5y+EGCyEOCmEOCWEmOMoPRQKR7Ju3Toef/xxzp49S3x8POfPnycoKIgdO3YUeU2bNm2Ij4/n1KlTAHz66af07du31DJTUlJo0KABzs7ObNu2jbNnC4/6GxUVxdtvvw1A+/btC+Tv37+fixcvAprBPnToEM2bN6d79+5s377dNvMnz+3Tu3dvVq1aBWjjGf7+/nh7FzTqAwcO5N1337V9P3DoeKnbpig9DjH+Qggj8D4wBGgPjBNCFPzrUihqOKtXr2bEiBH50kaNGsXq1auLvMbNzY0VK1YwZswYQkJCMBgMTJs2rdQyx48fT2xsLCEhIXzyySe0bdu20HIBAQG0a9eOJ554otD8q1evMnToUIKDgwkNDcXJyYmZM2dSv359li5dysiRIwkLC+ORRx4BYO7cuezbt4/Q0FDmzJlTYAwhj8WLFxMbG0toaCjtuz7AkuVfFFpOUT5E3uyBKhUqRA9grpRykPX7CwBSyr8XdU1oeEd56MCvVaShorZw/Phx2rVrV7aLKtDtU53JyMggJCSE/fv3U7du3coRUtK9LE/+nf5O5aizvLF98ub4t/EvRQW3CSvsb1kIsU9KGVHY5Y5y+zQGztt9T7Cm5UMIMVUIESuEiM2+XvLMCoVCUTFER0fTrl07nn766coz/DWQpk21Qw/CqvWAr5RyKbAUIKxVYNW/oigUtZT+/fsXORagKBoPD/0Ic1TP/wJg/8hqYk0rGmX6FQpFNefmTe3QgzBH9fx/AVoJIYLQjP5Y4NHiL1HWX6FQVG8uXdLO3t7VX5hDjL+U0iSEmAn8ABiB5VLKo47QRaFQKGojDvP5Sym/A75zlHyFQqGozegosJty+yhqHklJSYSHhxMeHk7Dhg1p3Lgx4eHh+Pj4FLqwqrzMnTuXN998s0zXFBVWISoqinXr1lWEWgoHoBvjbza4OVoFhaLC8fPz48CBAxw4cIBp06bx7LPP2r7nRcssjtKEe1AoCkM3xj/XqSpGUBSK6oPZbGbKlCl06NCBgQMHkpmZCUBkZCSzZ88mIiKCd955h3379tG3b186d+7MoEGDuGQdCFy8eDHt27cnNDSUsWPH2uo9duwYkZGRtGzZksWLF9vS//nPfxIcHExwcLAtrIM9UkpmzpxJmzZt6N+/P1evXq3cG6BDmjfXDj0Iq9bz/O1RTh9FlbC00MWQGr1fhHYjtc+ntkLs+KLLTo0ttypxcXGsXr2aZcuW8fDDD7N+/Xoee+wxAHJycoiNjSU3N5e+ffuyadMm6tevz5o1a3jppZdYvnw5Cxcu5MyZM7ZonnmcOHGCbdu2kZqaSps2bZg+fTqHDh1ixYoV7NmzBykl3bp1o2/fvnTs2NF23YYNGzh58iTHjh3jypUrtG/fnkmTJpW7nTUJt6p0UJRTmG6Mv0HmOloFhaJKCQoKIjw8HIDOnTsTHx9vy8uLl3Py5EmOHDnCgAEDAO1tITAwEIDQ0FDGjx/P8OHDGT58uO3aBx54AFdXV1xdXWnQoAFXrlxh586djBgxwhZdc+TIkezYsSOf8d++fTvjxo3DaDTSqFEj7rvvvkpsvT7Je8b6+FR/Ybox/q45SY5WQVEbKG2P/e77oNP0SlXl9nDOeW4fuBUCWUpJhw4d+Pnnnwtc/+2337J9+3a+/vprFixYwOHDhwutV40bVBxXrmjnKjH+5RSmG5+/KLmIQlHraNOmDYmJiTbjn5uby9GjR7FYLJw/f557772X119/nZSUFNLydn4qhN69e7Nx40YyMjJIT09nw4YN9O7dO1+ZPn36sGbNGsxmM5cuXWLbtm2V2jZF5aKbnr9y+isUBXFxcWHdunXMmjWLlJQUTCYTs2fPpnXr1jz22GOkpKQgpWTWrFn4FNND7NSpE1FRUXTt2hWAyZMn53P5AIwYMYKtW7fSvn17mjVrRo8ePSqzaYpKxiEhne+E8Jb+8sDv1xythqKGoUI6OxgV0jn/9bUgpLNCoVAoHIh+3D7K76NQKKo51j3rdSFMR8ZfoVAoqjcuLvoRphu3T4azn6NVUCgUimJJTtYOPQjTTc/fLJwdrYJCoVAUS2Kidvb1rf7CdNPzVygUCkXFoRvj72pKcbQKCkWlYDQaCQ8PJywsjE6dOvHTTz/dcV2vvPIK0dHRpS4fExPDgw8+eMfyCiM+Pp7g4OB8aYcPH7aFrvb19bWFrug/7Am++m4rCxcurFAdFCWjG7ePiznd0SooFJWCu7s7Bw4cAOCHH37ghRde4H//+98d1TVv3rwK1KziCAkJsbUxKiqKBx98kNGjR9vmzQ9TayaqHN30/BWK2sDNmzepV6+e7fuiRYvo0qULoaGhvPrqq7b0v/71r7Rp04ZevXoxbtw42wYt9hustGjRgldffZVOnToREhLCiRMnSq3H5s2b6dGjB506dWLMmDG20BDz5s2jS5cuBAcHM3XqVPIWie7bt4+wsDDCwsJ4//33y9Tmlav+w8yZM236T58+ne7du9OyZUtiduxh0lMv0q5dO6Kiogrq13skYyY8U2zoCkXh6Mj4q3n+iiogMrLg8cEHWl5Ghvb9gce1Iy9/5Uot/9q1gteWgszMTMLDw2nbti2TJ0/mL3/5C6AZuLi4OPbu3cuBAwfYt28f27dv55dffmH9+vUcPHiQ77//ntjYooPR+fv7s3//fqZPn17qHbyuXbvG/PnziY6OZv/+/URERPDPf/4TgJkzZ/LLL79w5MgRMjMz+eabbwB44oknePfddzl48GCpZBTH9evX+fnnn3nrrbcYNm4Gz86YyNGjRzl8+DAHDhzIr9+O/xDRMdimn6Np2VI79CBMN24fFdhNUVOxd/v8/PPPTJgwgSNHjrB582Y2b95si7GTlpZGXFwcqampPPTQQ7i5ueHm5sbQoUOLrHvkSG3/gc6dO/Of//ynVPrs3r2bY8eO0bNnT0DbOyAvjs+2bdt44403yMjIIDk5mQ4dOtC7d29u3LhBnz59AHj88cf5/vvv7+heAAwdOhQhBCEhIQTU9yekQxswGOjQoQPx8fEkJCTc0s+STU5OLj169r1jeRWJc1VOSiynMN0Yf9XxV1QJMTFF53l4aPlFxXfx9y/++lLQo0cPrl27RmJiIlJKXnjhBZ588sl8ZQrbZaso8sI3lyV0s5SSAQMGsHr16nzpWVlZzJgxg9jYWJo2bcrcuXPJysoqtS5l1dlgMODqemshk8FgwGQyYTQab+lXzeIsXbOGH/P3r/7CdOT2UShqPidOnMBsNuPn58egQYNYvny5zZ994cIFrl69Ss+ePfn666/JysoiLS3N5nqpKLp3786uXbs4deoUAOnp6fz22282Q+/v709aWpptbMHHxwcfHx927twJwKpVqypUn5L1y+C3336rVJmlJSlJO/QgTDc9f7PQjaoKRZnI8/mD1uv++OOPMRqNDBw4kOPHj9tcLl5eXnz22Wd06dKFYcOGERoaSkBAACEhIdStW/eO5W/ZsoUmTZrYvn/55ZesXLmScePGkZ2dDcD8+fNp3bo1U6ZMITg4mIYNG9KlSxfbNStWrGDSpEkIIRg4cOAd61Ia6tevf0u/zJuafn9bROvWrStVbk1DNyGdW7RuJ+N/O+5oNRQ1DL2GdE5LS8PLy4uMjAz69OnD0qVL6dSpk8P0uWNUSOf819eEkM5CiEVCiBNCiENCiA1CCB+7vBeEEKeEECeFEINKVaE+nlEKRZUwdepUwsPD6dSpE6NGjdKn4Vc4lMr0pfwIvCClNAkhXgdeAP4shGgPjAU6AI2AaCFEaymlubjKlO1XKG7x+eefO1oFhc6pNOMvpdxs93U3MNr6+SHgCyllNnBGCHEK6AoU3IHaDr/cC5Wip0KhUFQUd9+tH2FVNdtnEpA38bcxcN4uL8GaVgAhxFQhRKwQIlYvYxMKhaL2YjRqhx6ElavnL4SIBhoWkvWSlHKTtcxLgAko8/wvKeVSYClApxZ1lPVXKBTVmqtXtXODBtVfWLmMv5Syf3H5Qogo4EGgn7zVdb8ANLUr1sSaplAoFLrm+nXtXCXGv5zCKnO2z2DgT8AwKWWGXdZXwFghhKsQIghoBewtsb7KUVOhcDiODulct25dW2yh5557rsRrIiMji40nVJHE7NjDT3v2F5q3cuVKRN22RG+7db82btyIEMK2AK0oSnOfYmJiyvVbVHcqc7bPe4Ar8KMQAmC3lHKalPKoEGItcAzNHfRUSTN9NJTXR1EzcXRI5969e/PNN9+QmZlJx44dGTFihC2uj6OJ2bkXL08P7hkwrtD8kA6t+WL9d/R/6AkAVq9eTVhYWIn1luY+xcTE4OXlxT333FM2pXVCpfX8pZR3SymbSinDrcc0u7wFUsq7pJRtpJR3HgFKoahhODKks7u7O+Hh4Vy4oHlhiwrrbE9hZf773/8yZswYWxn7DWOmT59OREQEHTp0yNeewnSNj49nyfI1vPXBx4SHh7Njx44C8nv3iGDv/kPk5uaSlpbGqVOnbKulAea9/n6hIaiLvU+//U782QSWLFnCW2+9ZZOdmJjIqMeepkvkaLp06cKuXbsAmDt3LpMmTSIyMpL+/VvyySeLbfKHDx9O586d6dChA0uXLrWle3l58dJLLxEWFkb37t25cuUKANeuXmPmxJm28Nh5bx6fffYZXbt2JTw8nCeffBKzuRT95RLQTWyfVIOPo1VQ1AJKE9E58oGm2hFZIRGdq01I5+vXrxMXF0efPn2KDeucR1Fl+vfvz549e0hP1zZgWrNmDWPHjgVgwYIFxMbGcujQIf73v/9x6MjJInVt0aIF0yY9wrMzJnLgwAF69+5dQGchoH9kD3744Qc2bdrEsGHD8uXPnDK+0BDUxd6nxctp0bwJ06ZN49lnn7XJfuaZZ3j2qSh+iVnH+vXrmTx5su36EydO8MMPP/Dll3t5//3XyM3NBWD58uXs27eP2NhYFi9eTJI1Fk96ejrdu3fn4MGD9OnTh2XLlmn358UFdOnRhYMHD7J//346dOjA8ePHWbNmDbt27eLAgQMYjcYKiZ+km4A5mcLT0SooFJWCo0M679ixg7CwMOLi4pg9ezYNGzbkm2++KTKscx5FhX52cnJi8ODBfP3114wePZpvv/2WN954A4C1a9eydOlSTCYTly5d4tiJU4QGtym1roUxdtQDLP7oC1JSUvjHP/7B3/72N1veth17eOORWflCUBd2v/LJXvtZoXKio6M5duRX7YvBlZs3b9rehh544AFcXV3p3t2VwMAGXLlyhSZNmrB48WI2bNgAwPnz54mLi8PPzw8XFxfb21Dnzp358ccftXu6czevv/86oI0F1a1bl08//ZR9+/bZYillZmbSoEEDmDCh1PeoMHRj/JXHX1EVlCaiMzety1Rui+9SARGdHRLSOc/nf+bMGbp3787DDz9cZFhne4orM3bsWN577z18fX2JiIigTp06nDlzhjfffJNffvmFevXqERUVRZY1cFxpdS2Mrp1DOTx7Ph4eHvmCu2VlZTPjj/OI3fdriSGo88k2Fy7bYrGwO3oNbm6uBX77vOvt9Y+JiSE6Opqff/4ZDw8PIiMjbfKdnZ2xjoWW2F4pJRMnTuTvf/97Ke5G6dGN28fdovbwVdR8HBnSOSgoiDlz5vD6668XGdbZnuLK9O3bl/3797Ns2TKby+fmzZt4enpSt25drly5UqoNX+p4eZKaVvL//sKFC/P1+EEz/lAwBHVpqVOnDqmpqbbvAwcO5N1/3XoryHtbs+fyZciz4ykpKdSrVw8PDw9OnDjB7t27S5TZo3cPVq/QHqZms5mUlBT69evHunXruGqd15+cnMzZs2c1YZcvl6lN9ujG+Ne1XHe0CgpFpZDn8w8PD+eRRx7JF9L50UcfpUePHoSEhDB69GhSU1PzhXQeMmRIuUM62zNt2jS2b99Oenq6LWxyaGgoPXr0KDBgbB9a+fYyRqORBx98kO+//97m3ggLC6Njx460bduWRx99tFQzioYOuZcN30QXOeCbx5AhQ7j33nvzpfn4eDNl4hiCg4MZNGhQvhDUpWHo0KFs2LDBJnvx4sXE/nqE0HuG0b59e5YsWVLgmpQUsFi0z4MHD8ZkMtGuXTvmzJlD9+7dS5T54oIX2bNrDyEhIXTu3Jljx47Rvn175s+fz8CBAwkNDWXAgAFcunRJE5aSUqY22aObkM4RzT1lbHwqCN08rxQ6QIV0djAqpHP+66swpLNufP4ASIsy/goFWkjnY8eOkZWVxcSJE/Vp+BUORX/GX6FQqJDOinKjjL9CoVBUEKIq49CUU5gy/gqFQlFBVOk2wuUUpi8HujL+CoVCUSHoxvhfFI3AWa3yVSgU1ZeLF7VDD8J0Y/wlooodagpF1ZEXith+Ln18fDzBwcEO0Sc+Pt4W6K19+/ZMmDDBFq+mKL766isWLlxYbtn2Qdf0RmqqduhBmG6MvwrwoKjJrF69ml69ehUbTqEkyhISoTTcddddHDhwgMOHD5OQkMDatWuLLT9s2DDmzJlToTooKg/dGP/6MhGybjhaDUVtZ9UqCL4PfNpBixba93KSlpbGzp07+eijj/jiiy8KLRMfH0/v3r3p1KlTvg1fYmJi6N27N8OGaatOY2Ji6Nu3Lw899BAtW7Zkzpw5rFq1iq5duxISEsLp06cB+PLLLwkODiYsLIw+ffoUq5/RaKRr1662UM9ff/013bp1o2PHjvTv398WjnjlypXMnDmzyPrNZjPPP/+8LUT1v/71L0CLXTNz5kzatGlD//79bWEMFJWLbmb7uJAL5hxHq6GozaxaBVOnarGdAc6e1b4DjB9/x9Vu2rSJwYMH07p1a/z8/Ni3bx+dO3fOV6ZBgwb8+OOPuLm5ERcXx7hx42yhnPfv38+RI0cICgoiJiaGgwcPcvz4cXx9fWnZsiWTJ09m7969vPPOO7z77ru8/fbbzJs3jx9++IHGjRtz48aNYvXLyspiz549vPPOOwD06tWL3bt3I4Tg3//+N2+88Qb/+Mc/8l1TWP0fffQRdevW5ZdffiE7O5uePXsysMfd/HroGCdPnuTYsWNcuXKF9u3bM2nSpDu+n4rSoZueP6Bm+ygcy0sv3TL8eWRkaOnlYPXq1bbgZ2PHji3U9ZObm8uUKVMICQlhzJgxHDt2zJbXtWtXgoKCbN+7dOlCYGAgrq6u3HXXXQwcOBCAkJAQ4uPjAejZsydRUVEsW7asyI1BTp8+TXh4OAEBAQQGBhIaGgpAQkICgwYNIiQkhEWLFnH06NEC1xZW/+bNm/nkk08IDw+nW7duJCUlEXc6nu0/xTJu3DiMRiONGjXivvvuu4O7WD0wGrVDD8J00/MHlPFXOJZz58qWXgqSk5PZunUrhw8fRgiB2WxGCMGiRYvylXvrrbcICAjg4MGDWCwW3NzcbHmenvlnwdmHFzYYDLbvBoPBNi6wZMkS9uzZw7fffkvnzp3Zt28ffn5++erJ8/lfu3aNnj178tVXXzFs2DCefvpp/u///o9hw4YRExPD3LlzC7SrsPqllLz77rsMGjToVsGbJ/nux+13dO+qI3ffrR9hquevUJSWZs3Kll4K1q1bx+OPP87Zs2eJj4/n/PnzBAUFFYhgmZKSQmBgIAaDgU8//bTc2/idPn2abt26MW/ePOrXr8/58+eLLOvv78/ChQtt8eRTUlJo3LgxAB9//HGp6x80aBAffvihbdbQb7/9Rnp6Bn3uiWDNmjWYzWYuXbrEtm3bytU2RelQxl+hKC0LFmg7utjj4aGl3yGrV69mxIgR+dJGjRpVwPUzY8YMPv74Y8LCwjhx4kSB3n5Zef755wkJCSE4OJh77rmnxE3Phw8fTkZGBjt27GDu3LmMGTOGzp074+/vX+r6J0+eTPv27enUqRPBwcE8+eSTmExmRgwdQKtWrWxTSm/fMUxPJCRohx6E6Suk88Gj4NPC0aooahBlDum8ahW88CdIuKT1+BcsKNdgb61HhXTOf70K6VyQdNzVCl+F4xk/HoZa/5ccGM9foSgvunH7JEkf8KzvaDUUCoWiRlDpxl8I8UchhBRC+Fu/CyHEYiHEKSHEISGE2oVCoVAoqphKNf5CiKbAQMB+LtwQoJX1mAp8WJq6nMkFU1aF66hQKBQVhbOzduhBWGX3/N8C/kT+wDwPAZ9Ijd2AjxAisKSKAsU1uH6mktRUKBSK8tOypXboQVilGX8hxEPABSnlwduyGgP2k4oTrGmF1TFVCBErhNDWscvyzW1WKBQKhUa5jL8QIloIcaSQ4yHgReCV8tQvpVwqpYywTVXSybRUhaIsCCF47LHHbN9NJhP169fnwQcfLPa62NhYZs2aVWyZGzdu8MEHH1SInpVNfHw8TZo0wWLJv54nvNdw9uzZw+TJk/OFtbidV155hejoaADefvttMjIyK1Xfwjh3rlwLvqtUWLmmekop+xeWLoQIAYKAg0KLwd8E2C+E6ApcAJraFW9iTSuFQNXzV9Q8PD09OXLkCJmZmbi7u/Pjjz/aVtAWR0REBBERhU7htpFn/GfMmFFR6lYaLVq0oFmzZuzYsYO+ffsCcOK330lNTadbt25069at2OvnzZtn+/z222/z2EPd8fBwr1SdbyezKp835RRWKW4fKeVhKWUDKWULKWULNNdOJynlZeArYIJ11k93IEVKeamUFVeGugqFw7n//vv59ttvAW3V77hx42x5e/fupUePHnTs2JF77rmHk9bFPTExMba3g7lz5zJp0iQiIyNp2bIlixcvBmDOnDm2AG3PP/88AIsWLbKFVX711VcBrdfdrl07pkyZQocOHRg4cCCZVuOybNkyunTpQlhYGKNGjSLDGtwuKiqKWbNmcc8999CyZct8G7C8/vrrhISEEBYWZovxf/r0aQYPHkznzp3p3bt3vo1r8hg3bly+sNZfrP+WsaPuByAyMpLY2FjMZjNRUVEEBwcT0mMob72/0qbPunXrWLx4MRcvXuTeBydw74MT8pcPCeGtt94qz09VY3DEIq/vgPuBU0AG8ERpL5QWM2ovL0WlEh1ZchmTNbKnk0fx5QD6x5RK7NixY5k3bx4PPvgghw4dYtKkSbb4Pm3btmXHjh04OTkRHR3Niy++yPr16wvUceLECbZt20Zqaipt2rRh+vTpLFy4kCNHjnDgwAFAi6wZFxfH3r17kVIybNgwtm/fTrNmzYiLi2P16tUsW7aMhx9+mPXr1/PYY48xcuRIpkyZAsDLL7/MRx99xNNPPw3ApUuX2LlzJydOnGDYsGGMHj2a77//nk2bNrFnzx48PDxITk4GYOrUqSxZsoRWrVqxZ88eZsyYwdaN+Sf7Pfzww4SHh/Puu+/i5OTEmv98z5cfv52vzIEDB7hw4QJHjhyBmye5ceNmvvxZs2bxz3/+k23ffIK/Xz322ZeHEkNY1xaqxPhbe/95nyXw1J3VY1HGX1EjCQ0NJT4+ntWrV3P//ffny0tJSWHixInExcUhhChyO8UHHngAV1dXXF1dadCggW2TFXs2b97M5s2b6dixI6BtJBMXF0ezZs0ICgoiPDwcgM6dO9vCPx85coSXX36ZGzdukJaWli8q5/DhwzEYDLRv394mLzo6mieeeAIPaxwkX19f0tLS+OmnnxgzZozt2uzs7AL6BQQEEBwczJYtWwgICMDJyUhw+9b5yrRs2ZLff/+dp59+mgfuDWPgfT2Lu7X5yz/wgC3EdW1HN+EdLll8sfi11c+SZIU+KU1P/U5jxpTAsGHDeO6554iJiSEpKcmW/pe//IV7772XDRs2EB8fT2RkZKHX24dyNhqNhW7rKKXkhRde4Mknn8yXHh8fX+D6PLdPVFQUGzduJCwsjJUrVxITE1OozOLihFksFnx8fGxvIDby7qUdea6fgIAAxo16oEB+vXr1OHjwID/88ANLln/I2g3fs/zTgm9ChZZfsoS1a9eyfPnyIsuXB7vbUfmUU5hubGmWdEGq2D6KGsykSZN49dVXCQkJyZduH0J55cqVZaqzTp06pNpt8j1o0CCWL19OWloaABcuXChx28TU1FQCAwPJzc1lVSm2rRwwYAArVqywjQ0kJyfj7e1NUFAQX375JaA9KA4evH0WuMbIkSP57rvvWLNmDWMLMf7Xrl3DYrEwatQo5r/8DPsPFpwBpLU7vWD5+fPZv39/iW24U1q00I4qoZzCdNPzB7CoAV9FDaZJkyaFTt3805/+xMSJE5k/fz4PPFDQGBaHn58fPXv2JDg4mCFDhrBo0SKOHz9uC5vs5eXFZ599hrGYHaH++te/0q1bN+rXr0+3bt3yPUwKY/DgwRw4cICIiAhcXFy4//77+dvf/saqVauYPn068+fPJzc3l7FjxxI2+5EC1/v4+NCjRw8uX75My6CmBfIvXLjAE088oU0JtWTz91f/r0CZqVOnMnjUZBoFNuDtd5feKg+2fQlqO7oJ6dy6WX15+JftuAaUIfyuQlECZQ7pDJXm9qmV1LCQztZhkjvukJcppPNtwsoa0lk3bh9vkQE3S7ccQKFQKBxBdrZ26EGYbow/FD+gpFAoFIrSoy/jX859SxUKhUKhoS/jr8I7KBQKRYWgq9k+0qI2cFfYYc4l5/xenFzcMfi3BhevypeZlATZaVqokfhD0Lgx+PlVvlyFLnCvylBC5RSmL+Ovev41HykhJxVyM8BiBq+GSGEgJ9eM5doJ3IQJ4emPxZTLjW//jG/m7wAssTzKGee2eLg601/soq3pCCbv5ghPf4zuPjh5+uLq5YtL3UAMAcGaLIsJzDmQmwlCAAKkRVtJLgxgdAaD3b9I0jVIOA8NjGA0gjeQdhFkKvi3uFUu9RL5t7BAq9tg1PahdlHrVWoqzZrpR5hujH8OTlhKE0tFoRuyc01cPLQFjzPfYMxKwiUnGXfTDZy5tTJ1Ts4sLme5YZGSpz020sU5DtD8lb5AmqEuWS7++NRtRVPhQ3pWDq4pF/GT8ZAYD4n5ZZ41N+AN81TcXZyw5GQwpas/3Mi/SVBeCJEkWZcM4YlBCLzIoK68Do3sVlVabbjJks2lxBtIqT1DGpOCgSLeUj0Nt4x/ThqkXUb4tWL82DF8tupzMDhhMpkIDAykW7dufPPNN0Xev9jYWD755BNbELfCuHHjBp9//rkuonqCtojt+eefp3HjxmRlZfHkxBE8+1RUhcpYsmQJHh4eTJgwoULr1Ru6Mf4XzP5kNyo+hoeiGmMxY7p8iJTftvN7Zh223Awi7sI1gjjHy14/5yuaLVzJMXgAgq53NcDs1QgPF2f8Lp/gWooF19wbuJrTSPENx/+BBXi5+zDWvoKsjpiTT5F+6Tdy0pIwZ96AzBuQc5Mcox9d6jYhMzsXd+cAzCITk3BF66lLQCCFASElTk6uuAonLFIizBathBTkSgO50oglW4AACwJXbyeEEEgpSTXXQ8q88hKLxYIQAjcngROuuEiJEELbltScg6eHO0cOHyIz4TDu3r78GLOnVoZ0zuORRx7hvffeIykpiTat72b0Q4No2r7i1lRMmzat0HSTyYSTU/lM4u/ai2jV7OZVTmG6Mf4AFoua6qkrLCZyzu3lxuFv8LryEx6WNPyAhNwg0rwm06/j3QQ3605a+l24+wZi9AoAj/q4uniS178ema/CV/N9q1+UXLe6GBt1xrtR5wJZAWibR+dx/PhxnPzvKrSauvZfpBccTkLk5OAaiKZf3luFiws0s/8HvDX2IKUkO9fEzYxsErNzkTm5GNNS8HRzoY67Dy71vEAYuH9Qf76N/h+jhw5k9eerGPfQAHbsOw5oIZ2feeYZsrKycHd3Z8WKFbRp04aYmBjefPNNvvnmG+bOncu5c+f4/fffOXfuHLNnz2bWrFn5QjoPGDCARYsWsWjRItauXUt2djYjRozgtddeIz4+niFDhtCrVy9++uknGjduzKZNm3B3d2fZsmUsXbqUnJwc7r77bj799FM8PDyIiorC29ub2NhYLl++zBtvvMHo0aMBLaTzZ599hsFgYMiQISxcuJDTp0/z1FNPkZiYiIeHB8uWLaNto6JDNfr5+XF3y+ZcupJI0/bw2WefsXjxYnJycujWrRsffPABRsCrUSemTJnK5s2badiwIV988QX169dn2cq1LF25lhyzIZ/ec+fOxcvLi+eee47IyEjCw8PZuXMn48aN449//GOR+pSGImLuVQ7lFKYr4y8L+FEV1REpJVd2f4b30eV4WFJpYE2/6VSfzAYRtGodyfzW/eyuKLiE31FErowsOjM3V1tU42z9O8xB8/O4usL+wjfSjomKwc3FGTcXZywWSUZ2DulZOdzMyOJmRhaebi4AjJ04lXmvvcaDI8dy6Hgck56YaDP+bVu3YsePX+Pk6Uv0lq01OqSzPefOnSMrO5vQDm04fvw4a9asYdeuXTg7OzNjxgxWrVrFhOHdSE/PICIigrfeeot58+bx2muv8d577zFy2ACmRD0M3m0K6G1PTk4OsbGxRf/uNRTdGP8Wxsu4nfoKuox3tCqKwpCS7Mx0foq7QvSvcbS8cYI/eKSS4hxAbrN78Q19EG//NngLHQfldrYaeJmleYgMBq3X71y44b8dg0Hg5e6Kl7srZrOFlIwsUtKzkEDrtu2IP3uW1Ru+5/6hw8Gtnu26lMQEJj7zLHFnziEMRnLNhY8n1ISQzgBr1qxh+/btnDhxgvcWvYybmytbtvzIvn376NKlCwCZmZk0aNAA6IbBYOCRR7QYQXkPK4Ajx+J4ef473EjNLqC3PXnX1jZ0Y/wFaLM/FNWLnDRSDm5AHv2SfRmN+Tg9kmb1fWjV91Fy/B6ibtMu1pk0+iAmKqbkQhUQ28doNOBbxwMXJyNIuJScSp/7BvDHPz7H5ugfSU1JsZX9y7yF3NvnHjZ8vJj4cxeIHBEFN85CTnq+OmtKSOc8n39sbCwDB/Rn2P33IaVk4sSJBYOyFXK9detYoqa/wMbP3yes50MF9LbH07N2zr5Si7wUZUdK0o5+S+LnEzCt7EfdX9/BJ+cindzO8fLYSP46cSB9OnbApVlXXRl+R+Dl7ooQ0MDHi4lRT/D0H5/Hr1ELElPSyTWZyTWbSUnLoHGrMPAJYuW6/2oX5qZDRiKYit/HVY8hnfOIiIjg8bHDeOfDT+jXrx/r1q2z6ZqcnMzZs2cB7cGSt4Xk559/Tq9evTS909IJbFi/1HpXBJ6e2qEHYboy/ki1yMvRmJJOk7R6Al67XqV+2jEMWLjm2Yb0nnOpN3EjrZsG2HpeitLj6eZCWPvWvPrCn/Gv64nRIMg1W0hITGHStKf485w5dOx6DyajOxicwTNAOxvsXE4WkzZ91K4Hbh/S+fnnn2fgwIE8+uij9OjRg5CQEEaPHl1iiOa8kM49e/akbdu2JbZl8ODBDBs2jIiICMLDw3nzzTcBWLVqFR999BFhYWF06NCBTZs2lVjXn2dPZsWqDTRt2pT58+czcOBAQkNDGTBgAJcuaVt/e3p6sHfvXoKDg9m6dSuvvPKKpvdLz9DtvodLrXdF0KSJdjhEWHq6FuHTYNDOJTzwdBPSOaK5p9z8+Vv49pzqaFVqLUfiL/O/6E08ZVlGpvAkN3wy3iFDwc3H0ardMdU5pHOuyUxaVg5pmdmYzBYMQuDp5oKXuwuuzk63tjTNe9imXoKs62B00cYM3OrmX6RWHamAkM5ejTqRlpZeaF6x196JTpX825cppHNSEly4ADk5ABy/do12Q4bkKxIBxEpZaG+smv9l5EcvD6qaRsrpPXxyxMQvvyXQwKcJZ9s/S/OIB3F3q1vyxYo7xtnJSD0vd3w83cjKMZGWlU1aVjapmdk4Oxnx9tAGj22v70YX7U3AnAPpVyD9Krh6g3s9cHJXLrgq4NQp7Xz33ZUs6OxZSMy/gtEiDOzs1BvvtBQ6nDqCkDLfW+Dt6Mr4o2L7VC0WE0mbF+B37muyM8cyutf9DO7SRhukVFQZQgjcXZ1xd3XGr46F9KxcbmZkkXQzg5T0LHzreODh6ozw8AN3X831k3VdO2enaIdHffAscmWErkm7WHnbMpaVKgk8nJRkM/wSuOFdjww3D5JNsGzsUwB4pqeS6+wCL08sshrdGP8kWYd0/3BUCK2qwZJ6ietfPYdf+klMGJl8byt8Qts7Wq1KQeatuNUBBoOBOh6ueLm7kJljIvlmBldvpOHiZMTL3RVnJwOuzp4YXetobwCZ1yHrBrjWuVVJbgYIAzi5OawdijJym4sHwGQwYnJy5kYdH1yys/DISOOvb/2ZpID6JN1zFymXDRwopkrdGP+bFk8yvSv7XUoBkHFqO4atL+JHFukGb5wGLcKnacHVsjUBNzc3kpKS8PPz080DALS3AQ9XZ9z9vUnLzCElI4vk1AxbnreHK3U93LRV0571NWOfR9oVbZaQs4fmFnLxBqNuTEHNJSkJ0tPAIiG+6EVnErjuXQ/37EwMFguNLp8n1ZSN7+UTNH/YheatzOB8Cg7Gs7AYcbr6xZXPv/LJ/H0nzlv/hDMmrtWLwO+BvyE8fB2tVqXRpEkTEhISSLzNf1osWZe1s1v1ckNaLBbMFklWjonzuSYtnpCLE+4uzhgM1geblJB9UzP+9v9PRhdwdgOjmxZ9tKoo6V6WJ/9Of6e868ypcP16fl9O3jBXyhltVo2vb77plpetl5bZQ52eDklJXPbQfhNLIePXZoOBXCcXcp2dyTZdp97N6xilGTyNuJku0iRrDbT31h4ep9Mgo3gfVKUafyHE08BTgBn4Vkr5J2v6C8AfrOmzpJQ/lFRXHZGBy404aNqgpKKKOyAjO4c9J84TuPcD2mIiqckQ/IfMq/GDhM7OzgQFBZXtoujp2rl/TIXrU1EkXEvhq5+PsefEKZyMBrq3a0ZIi0BaNfHHt447IjcDzmyFM1sgYbc2TRTg3r9CK+uMEWnJ/8ZQGZR0L8uTv2oknDkDr+Ro4Y8XLIDxJUQIWLUKTkVBrgkWFJL/kvWcl+fiAsuXa/VKybovLSAt/OWFbO2e2s+ES7+queKAW7Fj0f7Hut4PJ+KZOcUfb4+WvH7YBJ5GzHVckV5OZPjXJSXAj7rGdD6/eS/dd26n13efI7r7whNBYLLAwRRIcIKWfWD1Bki6UWxTK22qpxDiXrRb9YCUMlsI0UBKeVUI0R5YDXQFGgHRQGtZwgquiOae8vuPXqN+/+cqRd/aTHpWDnM/+5Er19MY7HOGQU0y8Rv8auX/4+uV6EjtXI2Nfx6XklP57pcT/HLyPBnZWiAwTzcXmtX3oXmAD6FBgbRt4IZTwi6I3waRc29tirN9AVz/HVr2h6D7wCug4hUs6V6WlL+qnWbg/5Kt7bFgNt86326oQdt45513Cn8IrFoFTzwBf8pFAinv1OVs0yBuBNYnu54nri65eIVlE28J4MphX4Iu/E6Y2xka9nSGFg0QWTcQt4Xy/iDwHTxcXWkeUI+ux17GMzWu0GYkHhWc2u7C+49M5m6XG8z1KnqOvuX1Exh+t74ahPuAnytE/RUem1KgrBBin5Sy0NCvldnznw4slFJmA0gp85YRPgR8YU0/I4Q4hfYg+Lnwam4hVXiHCkemXeXwxre5nhLCc6MjCWnxsK5834riCfStwx8GdSFqQGfir1znzOVkzl+9wbnEG2w5cJr/xv5GHXdXBnRqRb9eC6jjYg3XICWc/0mbMnrlIPz8DwgI1R4CLfuDV8PKV37VKjizG7KyYbBTfsN+u4EHMJuRQLqLG1IYQEiuifrcbFsX15xs6vrn4hkIrv+dh1PW9xgbemsuMEB6NODE+wfYN+RRDnuF8Gzd7/F+M4MwkQMkWw+N0xnt+a1Fa3Z37Em88zGme3wLWVq+WQrMGDFjwCwN/H4xkdQsC1sPnsbbQ9LUWNfW5xcCDEJzEe29uzVrmkSSmxHH6cufkOXtgcwwY0jNxZiahfFmDiLdBDdzMVzJutXm33Jg6Xslv9EUQmUa/9ZAbyHEAiALeE5K+QvQGNhtVy7BmlYAIcRUYCpA52YeUNQGGYo7w2IiedNsumf8RsBdTgQFjXO0RopKwmgwcFegH3cF3povl51r4ujZK2w7eJr/7DrCpp+PEhoUyIBOrejQPAAxZi2c3wW/R8O5XXDlkHbsfhvueQ6CxxYtsDSsWgWndmkulgEif6981SqYOhWetQZ/s/rdLRYLCIHB+j0HZ9a6jWP/vG40MV4jwC2Fxi7JNDMk0tiYxJr0ERyb1ByAMa7bGeq2x1rfCbhwS5UkeZm/j3gS59wc2luO4CNScRNmpEUiMyyI1FxIN2FpZGFM+ic8/Pcb3PCqy9m2dxMd3hZj1HQaN23JjMkNMBoMrFmXhbOTkTfdXLBISfLNDFIzB5Cca8JoMODt4UYDH0+EEJgtFlqs/ILJXyznldZbADNu7xRz3wzWN3I3N21BwR0Yfiin8RdCRAOFdQFestbtC3QHugBrhRBl2nVASrkUWAqa20fN869Ykn5ciF/6b6QZ6tJi4DOOVkdRxbg6O9Hp7sZ0ursxCddS2HHkDD8fO8sbX/6PoIb1GNqtPZ1aDcBw10Btq0vbg2AnBITdqujQZ9pDod5dUK8l1AkEzwbg7lf04LHNxWIXfC4pCSZN0j6/+hIYc4hz6UC062As/ydwcpG4+BsxCyeOXGuEb0oyaV6evOH7OY9xtFAxD+euRb6fQraLK86t3bh4l5GMbBdSze4kDhtPUrYzuSYzbh4ezFy7ktCdP+L2fDZku8CbZkSmGWE/Lm73puGTloLPocMw+1noqW00ZcoBE+DjdWt/XYMQ+Nf1xL9u4XF4jAYDHSY9Cq4SdvwPsu3ebIS4NTB/u8sqMrLwe1tKymX8pZT9i8oTQkwH/iO1QYW9QggL4I/2vLUP4N6EfM/g4gQq419RpB7ahN/ZjZgx4Dz4DW2BkKLW0sS/LuMiwxndK4Rdx+L5Zs8JFm/aRSM/b4Z2a0doy0AMjfrgEdSP3OwMYo4mcHznToQQjEr9kcaZR7UBZHuEEVr2I6PXXFydnTBmp8B3M5FGFzJOHSP9xS6k+3ngJCy4Lcrlp6PNuJnshNO3u+g84i5at/KnFdCKGLgV4Zqbqc5YTnckua4vXtK6TiNXwoVM7UjIgIRMGJtJS4sZzlsvPG2nW/Pm8NFtC6DSs2CHde6JOQcy7PIMBs0/I7i1PXNx4wd3wvjxkLtM+7wipmLqLIbKdPtsBO4FtgkhWgMuwDXgK+BzIcQ/0QZ8WwF7S1OhiupZMWSe3oH77r8BkBo2A58mNXMOv6LsODsZiQy9i97BQfxyMoGvdx/jX9/tseW7OjvhZDSQnpVDQD0vnAwGlqb2IFA2p7EhicZOyQQ4Z+AjUvG03OTA6QT+uX8DRoOBVt7ZvChPIgDPQPDEDFjdOi5w+e6mHDDdTa6TMy5ezjS0/IqQJjwtqRiuWSDHApez8L6QydQfrJ7jl4B0Cp+ZU1Rf0cNDm/lzO3lGPG+2DxQ08HkD0PNjSndDqzGVafyXA8uFEEfQ9jyaaH0LOCqEWAscQ3tDeqqkmT421Dz/cmO6Fodhy59xwkxyi+H4di16+bei9mI0aNNDu7VtyuH4y1xKTsVisZCUmkF6Zg59QlrSrpk27doiJVdvpBF/5TpxV64TfeU6yWmZeLpCgzpujAkNIDPHRNKN63ye/Uc8naHNF8tpEXcIt5E5IM3wbzNPXvkVsizQvDly/gIs05djfEYLPV2ocS9S+TyXCVov3c/6VpucXPKUz/HjIdra+64BBr44Ks34SylzgMeKyFtA2X5O4s0BnL3rCSphslmt4vvDV2iU24zmjQLwH/BijZ/HrygfQghCgwIJDQossoxBCBrWq0PDenXo3rZZ6SpOt2g+/yHWfWjPWtNdXGDBAsT48RgFcOYP2myf22f53G7gmzcvaNQd0Et/8MEqE1VuYbpZ4SsRWHS2/UB14/TFJNbtu0iv9rPpPKizmsevcBylcbGUphdezdwwz1XlMqRyCtON8QewWJTb505JP3+IZd+fxreOO+P7dQJj6fadVSgqjVrkYqmO6Kbr18iYRONz6xythi7JTYzD6fvpTDGv4OnBoXi4ujhaJYWiRhIZWe4ZmFUmTDc9f1dycc285Gg1dEdu4m9kb5qGF9nUDWiOf7OmJV+kUChqPLox/kCBuBmK4sm5fAzz10/iJTNJ9mqP/7BFys+vUCgAHbl9ALXIqyyYc8j4759xl5kk1YvA9+FlavMOhUJhQ2fGXw34lpbkHR/ik3OJFOcA/Ea8A06ujlZJoVBUI3Tl9kFF9SwVWdk5nI07hC/gdt8ryvArFFXEww/rR5i+jD+q518SOSYzb2/8ieMpg3hp4CRaN+/maJUUilrDjBn6EaYb458m3Un2aqdW+BaDJe0qyzcf4ti5JKYO6Urr4DLuUKVQKMpFhjUYnIdH9RemG+OfaKnL2foDaOdoRaorpixu/GcGw9Jv0u6el+mlDL9CUeXcf792jomp/sJ0NeCrNnAvmmvfvYpvVjxergb6dApxtDoKhaKaoxvj7yJycc26WnLBWkj22T34X95CLk64D12McK9X8kUKhaJWoxvj39iQRHDCCkerUf2wmMjcZo3N32YczvVbO1ghhUKhB3Rj/AE1z78Qbv66Dp+cC9x08se35zRHq6NQKHSCbgZ8AYRa4VuAzEvH8AZMbUaq+fwKhYOJitKPMF0ZfxXeoSA3Pe8iLqc97XyVu0ehcDR6Mv76cvuowG4FuOjfm6WZDyAbdXG0KgpFrefaNe3QgzBd9fyF8vkXwGTSHohORp09xxWKGsjo0dq5Sub5l1OYviyGcvsUwCnjIo0NiTjLbEerolAodIRujP8l6UtswwmOVqPa0e7cCv5eZyUu1w45WhWFQqEjdGP8c6QL150bO1qNaocw5wJgdFax+hUKRenRjfFHgEX5/AtgsGjG36CmeSoUijJQaQO+QohwYAngBpiAGVLKvUIIAbwD3A9kAFFSyv0l1ecrbtIh+RugY2WprEuE1Iw/RrUpu0LhaKZP14+wypzt8wbwmpTyeyHE/dbvkcAQoJX16AZ8aD0XSx0yaJ76S+Vpq1Pyev7K+CsUjueRR/QjrDLdPhLwtn6uC1y0fn4I+ERq7AZ8hBCBpalQbeBeEIPq+SsU1Ybz57VDD8Iqs+c/G/hBCPEm2kPmHmt6Y8Be4wRr2qXbKxBCTAWmAnRu5qFi+xSCUfX8FYpqw+OPa+cqmedfTmHlMv5CiGigYSFZLwH9gGellOuFEA8DHwH9y1K/lHIpsBQgormnVD3/gnzlM5WUlOtMc/d1tCoKhUJHlMv4SymLNOZCiE+AZ6xfvwT+bf18AWhqV7SJNa1EhNrDtwCXDI247uQDRmdHq6JQKHREZfr8LwJ9rZ/vA+Ksn78CJgiN7kCKlLKAy6cwVFTPgphMZpyMRkeroVAodEZl+vynAO8IIZyALKy+e+A7tGmep9Cmej5RmspyceKGUwPqVIamekVa6J/+JSbhQhk9agqFopZTacZfSrkT6FxIugSeKmt9V0QDvvF7iqqcRlvtMefQKXcPJn3F51Moaix//KN+hOnKaqjJPrdhDe1gFk76+iEVihrK0KH6Eaaf8A6o8A4FMGuRPM1CDfYqFNWBkye1Qw/CdNNhbCwvMunqi0CMo1WpPtj1/BUKheN58kntXCXz/MspTDc9fwEYpNnRalQvrD1/i+r5KxSKMqIb4w9qnn8BrD1/i+r5KxSKMqIv46/m+efHYOScpQGprqUKjaRQKBQ2dNVlVD3/26jXklfSoniwQzuaOVoXhUKhK3Rm/FXP3x6zxYJFSrV5u0JRTXj5Zf0I05nxVz1/e0y52gC4szL+CkW1oH9VLrQvpzDdWI2bBh9+dHvI0WpUKyxntrDS+006nXvf0aooFArgwAHt0IMw3fT804UX+116MNjRilQjzKYcDEJiELp5hisUNZrZs7VzlczzL6cw3VgNIcBiUW4feyy52jx/Fc5ZoVCUFd0Yf3eZQWjOXrCohV55WHKztA9GV8cqolAodIdujL+3OZlhOevBYnK0KtUGiylH+6C2cFQoFGVEN8ZfC/AAqBk/NqRJc/sIJ2X8FQpF2dDNgK8N5faxYckz/qrnr1BUC/72N/0I04/xz+v4q56/jRTfCL7LvMK9Dbs5WhWFQgHcc49+hOnP7aN6/jZSvVqyJacjZv8OjlZFoVAAP/2kHXoQppuevyzkU20n16yFu1ArfBWK6sGLL2rnKpnnX05h+rMaKrKnDY9rB+njfBjXrMuOVkWhUOgM3Rj/JKfGzBGvgls9R6tSbah/JZrJHv/FLfm4o1VRKBQ6QzfGHyEwYdCW+io0rJu5GJ3VbB+FQlE29GP8Aak2cM+PzfirFb4KhaJs6GbA19uUyCy5DNK7gmcDR6tTLRAWbYWv0dndwZooFAqAt9/Wj7By9fyFEGOEEEeFEBYhRMRteS8IIU4JIU4KIQbZpQ+2pp0SQswprSwnmUtzcQHMOeVRuUYhLKrnr1BUJ8LDtUMPwsrr9jkCjAS22ycKIdoDY4EOwGDgAyGEUQhhBN4HhgDtgXHWsqVHzfO3YbD2/IWTMv4KRXUgOlo79CCsXG4fKeVxAFFwEPYh4AspZTZwRghxCuhqzTslpfzdet0X1rLHyiC1PCrXKERekDsV0lmhqBbMn6+dq2RHr3IKq6wB38bAebvvCda0otILRQgxVQgRK4SIteQN9qqev42vAl/k6dwXwK+1o1VRKBQ6o8SevxAiGmhYSNZLUspNFa/SLaSUS4GlAB1b+Fitv+r555FrsWA0OoPayUuhUJSREo2/lPJO3ikuAE3tvjexplFMevHkeZZUz99GrtmCs9HoaDUUCoUOqawu41fAWCGEqxAiCGgF7AV+AVoJIYKEEC5og8JflabCHIMHO02h4OpdSSrrj4cS3+ZplkFOmqNVUSgUOqNcA75CiBHAu0B94FshxAEp5SAp5VEhxFq0gVwT8JSU0my9ZibwA2AElkspj5ZGVqaTDytz7qdXncDyqFyjaJB7HmdylNtHoagm/Otf+hFW3tk+G4ANReQtABYUkv4d8F1ZZQkEaoFvfozkzfZR4R0UiupAmzb6EaabLqNB5tKYK5Cb4WhVqgcWEwYsSAQI5fdXKKoDX3+tHXoQppvwDp6515jnuQKSekLDcEer43iscX1MOOGsgt0pFNWCf/xDOw8dWv2F6abnb4vmqWb7aFjDXJiFbp7fCoWiGqEf429Fqs1cNGzGX63uVSgUZUd33UaLxYzycANGF36SnfHyqEuoo3VRKBS6Q0fGX7l98uFWl9Wm++nk10gZf4VCUWZ0ZPw1VM//FrlmM05O6m4oFNWFTz/VjzDdGX8sarI/ADnpNJIXqGeu42hNFAqFlaZNSy5TXYTpZsA3y8WP19LGY6of7GhVqgXy2glecf+Y7leWO1oVhUJhZc0a7dCDMN30/C0GZ86YG2KRErJSCi/k4gkGa5NyM4ve9ctgBBevW9+Lqg/AyQ3yNksx52j1FoWr960pqTlpRY9PGJ3B2UP7bDFBTnrRdRbRJktGMkZAGtRsH4WiuvDhh9r5kUeqvzDdGH+AxoZreK4ZXGT+EuOTnDdor0JDzV/T3bKn0HKXaMh7zjNt3+flvoKRwqeQfmV4kD3G7gB0tuxjpLnQaBYA/MXpNSzW1bZPmpbQTCYUWm6f6MR/nEYC0FBe4mnT+2VuU56nX6qNXBQKxR2gG+Pv7uJChxaBZCV7FFnGx9uTbGetR++c5kFWZuFlLU4eNKh3q+efneiOoYh9Ajw8PWngrpX1yPIkK7Vo+fXreSGtxl9e9yTLVHhZo5snDepoddYzeZF1/c7aZBEGnFsNKupShUKhKBLdGH9nJwOPjhoNjC6yzNh833oVWa4x8Ey+lP8VWba/9bhV5+wiy84qpfxw63GLMUWWLalNRT82FAqFomh0M+CrUCgUiopDNz1/hUKhqO6sW6cfYcr4KxQKRQXh768fYcrto1AoFBXEypXaoQdhyvgrFApFBaGMv0KhUCiqNcr4KxQKRS1ESJ3sih4RESFjY2MdrYZCoVAUSWSkdo6JqR7ChBD7pJQRheWpnr9CoVDUQtRUT4VCoaggvvtOP8LK1fMXQowRQhwVQliEEBF26QOEEPuEEIet5/vs8jpb008JIRYLkRcGU6FQKPSNh4d26EFYed0+R4CRwPbb0q8BQ6WUIcBEwH7LmQ+BKUAr61F0mE6FQqHQER98oB16EFYu4y+lPC6lPFlI+q9SyovWr0cBdyGEqxAiEPCWUu6W2kjzJ8Dw8uigUCgU1YW1a7VDD8KqYsB3FLBfSpmNFlDTPsh9gjWtUIQQU4UQsUKI2MTExEpWU6FQKGoPJQ74CiGigYaFZL0kpdxUwrUdgNeBgXeinJRyKbAUtKmed1KHQqFQKApSovGXUvYvqUxhCCGaABuACVLK09bkC0ATu2JNrGkKhUKhqEIqxe0jhPABvgXmSCl35aVLKS8BN4UQ3a2zfCYAxb49KBQKhaLiKdcKXyHECOBdoD5wAzggpRwkhHgZeAGIsys+UEp51ToldCXgDnwPPC1LoYQQIhE4e8fK3hn+aDOXaiI1qW01qS321LR21bT22FNd29ZcSlm/sAzdhHdwBEKI2KKWRuudmtS2mtQWe2pau2pae+zRY9tUeAeFQqGohSjjr1AoFLUQZfyLZ6mjFahEalLbalJb7Klp7app7bFHd21TPn+FQqGohaiev0KhUNRClPFXKBSKWkiNMv5CiKZCiG1CiGPWUNPPWNN9hRA/CiHirOd61vTxQohD1hDTPwkhwuzqGiyEOGkNPT2nGJkTrfXGCSEm2qUvEEKcF0Kk1aS2CSE8hBDfCiFOWPVYqNe2WNP/K4Q4aNVjiRDCWNb2VMd22eV/JYQ4ovf2CCFirNcfsB4N7qRN1bRtLkKIpUKI36z/V6PK07ZSI6WsMQcQCHSyfq4D/Aa0B95AW20MMAd43fr5HqCe9fMQYI/1sxE4DbQEXICDQPtC5PkCv1vP9ayf8+rrbtUnrSa1DfAA7rWWcQF2AEP02BZrnrf1LID1wFi9/0Z2+SOBz4Ejem8PEANE1FBb8Row3/rZAPhXVDuLvQdVIcRRB1roiAHASSDQ7kc/WUjZesAF6+cewA92eS8ALxRyzTjgX3bf/wWMu61MhRj/6tg2a/o7wBS9twVwBr4GHqkJvxHgBexEM2h3ZPyrWXtiqEDjX83adh7wrKy2FXXUKLePPUKIFkBHYA8QILW4QgCXgYBCLvkDWrgJ0MJMn7fLKyr0dGnLVSjVpW1Ci+E0FNhSpgbkr6MFDm6LEOIH4CqQCqwrcyMKoRq066/AP4CMO1C/ANWgPQArrC6fvwhRcTsAOrJt1v8hgL8KIfYLIb4UQhQms8KpkcZfCOGF9go/W0p50z5Pao9aeVv5e9F+0D9XmZJ3SHVpmxDCCVgNLJZS/n6HdVSLtkgpB6H18lyB+0ooXiKObpcQIhy4S0q5oYLqqw6/03ip7QzY23o8XhGVVoO2OaFFN/5JStkJ+Bl4s4LqLpYaZ/yFEM5oP+YqKeV/rMlXhLaLGNbzVbvyocC/gYeklEnW5AtAU7tqmwAXhBDd7AachhVVrjLaZdW1OrVtKRAnpXy7BrQFKWUW2qv/Q3fSnmrWrh5AhBAiHs3101oIEaPj9iClzDunoo1jdL2T9lTDtiWhvZ3lyf8S6FTetpWKqvYzVeaBNmj3CfD2bemLyD+I84b1czPgFHDPbeWd0AZkgrg1iNOhEHm+wBk0H2A962ff28pU1IBvtWkbMB/tn8ag57ag+cUD7epaA8ysCb+RXZkW3PmAb7Voj/V6f2sZZzTX3LQa9P/0BXCf9XMU8GV52lbqe1AVQqrqAHqhvaYdAg5Yj/sBPzS/dBwQbXfT/w1ctysba1fX/WgzAE6j7VpWlMxJ1j+KU8ATdulvoPn1LNbz3JrQNrQeiwSO29U9WadtCQB+sepxBC08uZPef6Pb8ltw58a/WrQH8AT2WfU4ijbJwFgT/p+s6c2B7VZdtgDNytO20h4qvINCoVDUQmqcz1+hUCgUJaOMv0KhUNRClPFXKBSKWogy/gqFQlELUcZfoVAoaiHK+CsUCkUtRBl/hUKhqIX8P4cKKsEZtPy+AAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "benchmark = Benchmark()\n",
    "evaluator = MaintenanceCostEvaluator(dataset_id='turbofan_engine')\n",
    "detector = MultiModelHTMDetector(evaluator=evaluator)\n",
    "\n",
    "turbofan_engine_dataset = benchmark.get_dataset('turbofan_engine')\n",
    "rtf = turbofan_engine_dataset.get_rtf('FD001_1_0')\n",
    "df_score = rtf.score(detector=detector)\n",
    "detector.plot(df_score)\n",
    "evaluation = evaluator.evaluate_rtf(df_score, plot=True)\n",
    "plt.legend()\n",
    "print(evaluation)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}