analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ndV0dmyzhpHE"
   },
   "source": [
    "# Analysis of Bioacoustic Data\n",
    "\n",
    "This notebook provides tools for analyzing data using a custom classifier (developed with `agile_modeling.ipynb`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "BN69E14wNBUW"
   },
   "outputs": [],
   "source": [
    "#@title Installation. { vertical-output: true }\n",
    "#@markdown You will likely need to work with `embed_audio.ipynb` and/or\n",
    "#@markdown `agile_modeling.ipynb` before working with this notebook.\n",
    "#@markdown\n",
    "#@markdown Run this notebook in Google Colab by following\n",
    "#@markdown [this link](https://colab.research.google.com/github/google-research/perch/blob/main/agile_modeling.ipynb).\n",
    "#@markdown\n",
    "#@markdown Run this cell to install the project dependencies.\n",
    "%pip install git+https://github.com/google-research/perch.git\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "i984ftjPcxDu"
   },
   "outputs": [],
   "source": [
    "#@title Imports. { vertical-output: true }\n",
    "\n",
    "import collections\n",
    "from etils import epath\n",
    "from ml_collections import config_dict\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from chirp.inference import colab_utils\n",
    "colab_utils.initialize(use_tf_gpu=True, disable_warnings=True)\n",
    "\n",
    "from chirp.inference import baw_utils\n",
    "from chirp.inference import call_density\n",
    "from chirp.inference import tf_examples\n",
    "from chirp.inference.search import bootstrap\n",
    "from chirp.inference.search import search\n",
    "from chirp.inference.search import display\n",
    "from chirp.inference.classify import classify\n",
    "from chirp.projects.zoo import zoo_interface\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "TRETHuu1h7uZ"
   },
   "outputs": [],
   "source": [
    "#@title Basic Configuration. { vertical-output: true }\n",
    "\n",
    "data_source = 'filesystem'  #@param['filesystem', 'a2o'] {type:'string'}\n",
    "baw_auth_token = '' #@param\n",
    "\n",
    "#@markdown Define the model: Usually perch or birdnet.\n",
    "model_choice = 'perch'  #@param {type:'string'}\n",
    "#@markdown Set the base directory for the project.\n",
    "working_dir = '/tmp/agile'  #@param {type:'string'}\n",
    "\n",
    "# Set the embedding and labeled data directories.\n",
    "labeled_data_path = epath.Path(working_dir) / 'labeled'\n",
    "custom_classifier_path = epath.Path(working_dir) / 'custom_classifier'\n",
    "\n",
    "# The embeddings_path should be detected automatically, but can be overridden.\n",
    "embeddings_path = ''\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "ake6Xk_Hh-nN"
   },
   "outputs": [],
   "source": [
    "#@title Load Existing Project State and Models. { vertical-output: true }\n",
    "\n",
    "if data_source == 'a2o':\n",
    "  embedding_config = baw_utils.get_a2o_embeddings_config()\n",
    "  bootstrap_config = bootstrap.BootstrapConfig.load_from_embedding_config(\n",
    "      embedding_config=embedding_config,\n",
    "      annotated_path=labeled_data_path,\n",
    "      embeddings_glob='*/embeddings-*')\n",
    "  embeddings_path = embedding_config.output_dir\n",
    "elif (embeddings_path\n",
    "      or (epath.Path(working_dir) / 'embeddings/config.json').exists()):\n",
    "  if not embeddings_path:\n",
    "    # Use the default embeddings path, as it seems we found a config there.\n",
    "    embeddings_path = epath.Path(working_dir) / 'embeddings'\n",
    "  # Get relevant info from the embedding configuration.\n",
    "  bootstrap_config = bootstrap.BootstrapConfig.load_from_embedding_path(\n",
    "      embeddings_path=embeddings_path,\n",
    "      annotated_path=labeled_data_path)\n",
    "  baw_auth_token = ''\n",
    "else:\n",
    "  raise ValueError('No embedding configuration found.')\n",
    "\n",
    "project_state = bootstrap.BootstrapState(\n",
    "    bootstrap_config, baw_auth_token=baw_auth_token)\n",
    "\n",
    "cfg = config_dict.ConfigDict({\n",
    "    'model_path': custom_classifier_path,\n",
    "    'logits_key': 'custom',\n",
    "})\n",
    "logits_head = zoo_interface.LogitsOutputHead.from_config(cfg)\n",
    "model = logits_head.logits_model\n",
    "class_list = logits_head.class_list\n",
    "print('Loaded custom model with classes: ')\n",
    "print('\\t' + '\\n\\t'.join(class_list.classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "Ur03VoLyuBHR"
   },
   "outputs": [],
   "source": [
    "#@title Write classifier inference CSV. { vertical-output: true }\n",
    "\n",
    "#@markdown This cell writes detections (locations of audio windows where\n",
    "#@markdown the logit was greater than a threshold) to a CSV file.\n",
    "\n",
    "output_filepath = epath.Path(working_dir) / 'inference.csv'  #@param\n",
    "\n",
    "#@markdown Set the default detection thresholds, used for all classes.\n",
    "#@markdown To set per-class detection thresholds, modify the code below.\n",
    "#@markdown Keep in mind that thresholds are on the logit scale, so 0.0\n",
    "#@markdown corresponds to a 50% model confidence.\n",
    "default_threshold = 0.0  #@param\n",
    "if default_threshold is None:\n",
    "  # In this case, all logits are written. This can lead to very large CSV files.\n",
    "  class_thresholds = None\n",
    "else:\n",
    "  class_thresholds = collections.defaultdict(lambda: default_threshold)\n",
    "  # Set per-class thresholds here.\n",
    "  class_thresholds['my_class'] = 1.0\n",
    "\n",
    "#@markdown Classes to ignore when counting detections.\n",
    "exclude_classes = ['unknown']  #@param\n",
    "\n",
    "#@markdown The `include_classes` list is ignored if empty.\n",
    "#@markdown If non-empty, only scores for these classes will be written.\n",
    "include_classes = []  #@param\n",
    "\n",
    "embeddings_ds = tf_examples.create_embeddings_dataset(\n",
    "    embeddings_path, file_glob='embeddings-*')\n",
    "\n",
    "classify.write_inference_csv(\n",
    "    embeddings_ds=embeddings_ds,\n",
    "    model=logits_head,\n",
    "    labels=class_list.classes,\n",
    "    output_filepath=output_filepath,\n",
    "    threshold=class_thresholds,\n",
    "    embedding_hop_size_s=bootstrap_config.embedding_hop_size_s,\n",
    "    include_classes=include_classes,\n",
    "    exclude_classes=exclude_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GdJmpn0XzMj6"
   },
   "source": [
    "## Call Density Estimation\n",
    "\n",
    "See 'All Thresholds Barred': https://arxiv.org/abs/2402.15360"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "-lhqypjsu2L9"
   },
   "outputs": [],
   "source": [
    "#@title Validation and Call Density. { vertical-output: true }\n",
    "\n",
    "target_class = 'my_class'  #@param {type:'string'}\n",
    "\n",
    "#@markdown Bin bounds for validation. Should be an ordered list, beginning with\n",
    "#@markdown 0.0 and ending with 1.0.\n",
    "quantile_bounds = [0.0, 0.9, 0.99, 1.0]  #@param\n",
    "#@markdown Number of validation samples per bin.\n",
    "samples_per_bin = 25  #@param\n",
    "\n",
    "quantile_bounds = np.array(quantile_bounds)\n",
    "top_k = call_density.get_random_sample_size(quantile_bounds, samples_per_bin)\n",
    "\n",
    "embeddings_ds = project_state.create_embeddings_dataset(shuffle_files=True)\n",
    "results, all_logits = search.classifer_search_embeddings_parallel(\n",
    "    embeddings_classifier=logits_head,\n",
    "    target_index=class_list.classes.index(target_class),\n",
    "    random_sample=True,\n",
    "    top_k=top_k,\n",
    "    hop_size_s=bootstrap_config.embedding_hop_size_s,\n",
    "    embeddings_dataset=embeddings_ds,\n",
    ")\n",
    "combined_results = call_density.prune_random_results(\n",
    "    results, all_logits, quantile_bounds, samples_per_bin)\n",
    "\n",
    "ys, _, _, = plt.hist(all_logits, bins=100, density=True)\n",
    "value_bounds = np.quantile(all_logits, quantile_bounds)\n",
    "for q in value_bounds:\n",
    "  plt.plot([q, q], [0.0, np.max(ys)], 'k:', alpha=0.75)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "It0gpV_avg7h"
   },
   "outputs": [],
   "source": [
    "#@title Display Results. { vertical-output: true }\n",
    "\n",
    "samples_per_page = 40  #@param\n",
    "page_state = display.PageState(\n",
    "    np.ceil(combined_results.top_k / samples_per_page))\n",
    "\n",
    "display.display_paged_results(\n",
    "    combined_results,\n",
    "    page_state, samples_per_page,\n",
    "    project_state=project_state,\n",
    "    embedding_sample_rate=project_state.embedding_model.sample_rate,\n",
    "    exclusive_labels=True,\n",
    "    checkbox_labels=[target_class, f'not {target_class}', 'unsure'],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "A30usazfu8h8"
   },
   "outputs": [],
   "source": [
    "#@title Collate results and write validation log. { vertical-output: true }\n",
    "\n",
    "validation_examples = call_density.convert_combined_results(\n",
    "    combined_results=combined_results,\n",
    "    target_class=target_class,\n",
    "    quantile_bounds=quantile_bounds,\n",
    "    value_bounds=value_bounds)\n",
    "validation_log_path = call_density.write_validation_log(\n",
    "    validation_examples,\n",
    "    working_dir,\n",
    "    target_class)\n",
    "print('wrote log to : ', validation_log_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "uZHVzPttwGZ2"
   },
   "outputs": [],
   "source": [
    "#@title Estimate Call Density and ROC-AUC. { vertical-output: true }\n",
    "\n",
    "validation_examples = call_density.load_validation_log(validation_log_path)\n",
    "density_ev , density_samples = call_density.estimate_call_density(\n",
    "    validation_examples)\n",
    "\n",
    "# Plot call density estimate.\n",
    "plt.figure(figsize=(10, 5))\n",
    "xs, ys, _ = plt.hist(density_samples, density=True, bins=25, alpha=0.25)\n",
    "plt.plot([density_ev, density_ev], [0.0, np.max(xs)], 'k:', alpha=0.75,\n",
    "         label='density_ev')\n",
    "\n",
    "low, high = np.quantile(density_samples, [0.05, 0.95])\n",
    "plt.plot([low, low], [0.0, np.max(xs)], 'g', alpha=0.75, label='low conf')\n",
    "plt.plot([high, high], [0.0, np.max(xs)], 'g', alpha=0.75, label='high conf')\n",
    "\n",
    "plt.xlim(0.0, 1.0)\n",
    "plt.xlabel('Call Rate (q)')\n",
    "plt.ylabel('P(q)')\n",
    "plt.title(f'Call Density Estimation ({target_class})')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "print(f'EV Call Density: {density_ev:.4f}')\n",
    "print(f'(Low/EV/High) Call Density Estimate: ({low:5.4f} / {density_ev:5.4f} / {high:5.4f})')\n",
    "\n",
    "roc_auc_estimate = call_density.estimate_roc_auc(validation_examples)\n",
    "print(f'Estimated ROC-AUC : {roc_auc_estimate:5.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "0ii9H72iQknv"
   },
   "outputs": [],
   "source": [
    "#@title Display Logged Validation Examples. { vertical-output: true }\n",
    "\n",
    "validation_results = search.TopKSearchResults(top_k=len(validation_examples))\n",
    "for v in validation_examples:\n",
    "  validation_results.update(v.to_search_result(\n",
    "      target_class, project_state.embedding_model.sample_rate))\n",
    "\n",
    "samples_per_page = 40  #@param\n",
    "page_state = display.PageState(\n",
    "    np.ceil(combined_results.top_k / samples_per_page))\n",
    "\n",
    "display.display_paged_results(\n",
    "    validation_results,\n",
    "    page_state, samples_per_page,\n",
    "    project_state=project_state,\n",
    "    embedding_sample_rate=project_state.embedding_model.sample_rate,\n",
    "    exclusive_labels=True,\n",
    "    checkbox_labels=[target_class, f'not {target_class}', 'unsure'],\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "private_outputs": true,
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}