02_c_GenerateDetections&SearchThreshold.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import src.data.Dataset as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PYTHONPATH: :/Users/carlos/WORKSPACE/MegaClassifier/data/external/ai4eutils:/Users/carlos/WORKSPACE/MegaClassifier/data/external/CameraTraps:/Users/carlos/WORKSPACE/MegaClassifier/data/external/yolov5\n"
     ]
    }
   ],
   "source": [
    "required_paths = [\"/ai4eutils\", \"/CameraTraps\", \"/yolov5\"]\n",
    "python_path = os.environ.get(\"PYTHONPATH\", \"\")\n",
    "root_path = os.getcwd()\n",
    "\n",
    "for path in required_paths:\n",
    "    if not any(p.endswith(path) for p in python_path.split(\":\")):\n",
    "        python_path += f\":{root_path}/data/external{path}\"\n",
    "\n",
    "os.environ[\"PYTHONPATH\"] = python_path\n",
    "\n",
    "!echo \"PYTHONPATH: $PYTHONPATH\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The file /Users/carlos/WORKSPACE/MegaClassifier/data/interim/10000Images_filtered.csv has been successfully opened.\n",
      "\n",
      "OUTPUT_PATH:       /Users/carlos/WORKSPACE/MegaClassifier/resources/json\n",
      "MODEL_PATH:        /Users/carlos/WORKSPACE/MegaClassifier/models/MegaDetector/md_v5a.0.0.pt\n",
      "DATASET_PATH:      /Users/carlos/WORKSPACE/MegaClassifier/dataset/emptyNonEmptyDatasetFiltered\n",
      "OUTPUT_FILE_PATH:  /Users/carlos/WORKSPACE/MegaClassifier/resources/json/detections.json\n",
      "TRAIN_CSV:         /Users/carlos/WORKSPACE/MegaClassifier/data/interim/train/10000Train.csv\n",
      "REPORT_CSV:        /Users/carlos/WORKSPACE/MegaClassifier/reports/model_coberture\n"
     ]
    }
   ],
   "source": [
    "initial_threshold = float(0.025)\n",
    "\n",
    "positives_coberture = 0.0\n",
    "\n",
    "OUTPUT_PATH = os.path.abspath(\"./resources/json\")\n",
    "MODEL_PATH = os.path.abspath(\"./models/MegaDetector/md_v5a.0.0.pt\")\n",
    "\n",
    "DATASET_PATH = os.path.abspath(\"./dataset/emptyNonEmptyDatasetFiltered\")\n",
    "CSV_PATH = os.path.abspath(\"./data/interim/10000Images_filtered.csv\")\n",
    "dataset = dt.load_from_csv(CSV_PATH)\n",
    "\n",
    "dataset[\"file_name_abspath\"] = dataset[\"file_name\"].apply(\n",
    "    lambda x: os.path.join(DATASET_PATH, x)\n",
    ")\n",
    "\n",
    "IMAGES_PATH_JSON = os.path.join(OUTPUT_PATH, \"dataset_file_paths.json\")\n",
    "OUTPUT_FILE_PATH = os.path.join(OUTPUT_PATH, \"detections.json\")\n",
    "CHECKPOINT_PATH = os.path.join(OUTPUT_PATH, \"checkpoint.json\")\n",
    "CHECKPOINT_FREQ = int(round(len(dataset[\"file_name_abspath\"]) / 8, 0))\n",
    "\n",
    "TRAIN_CSV = os.path.abspath(\"./data/interim/train/10000Train.csv\")\n",
    "\n",
    "REPORT_CSV = os.path.abspath(\"./reports/model_coberture\")\n",
    "\n",
    "os.makedirs(OUTPUT_PATH, exist_ok=True)\n",
    "\n",
    "with open(IMAGES_PATH_JSON, \"w\") as f:\n",
    "    json.dump(dataset[\"file_name_abspath\"].tolist(), f, indent=1)\n",
    "\n",
    "print()\n",
    "print(f\"OUTPUT_PATH:       {OUTPUT_PATH}\")\n",
    "print(f\"MODEL_PATH:        {MODEL_PATH}\")\n",
    "print(f\"DATASET_PATH:      {DATASET_PATH}\")\n",
    "print(f\"OUTPUT_FILE_PATH:  {OUTPUT_FILE_PATH}\")\n",
    "print(f\"TRAIN_CSV:         {TRAIN_CSV}\")\n",
    "print(f\"REPORT_CSV:        {REPORT_CSV}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = round(initial_threshold * 2, 3)\n",
    "load_from_checkpoint = False\n",
    "while positives_coberture != 1.0:\n",
    "    threshold = round(threshold / 2, 3)\n",
    "\n",
    "    print(f\"Iniciando ejecucion con umbral: {threshold}\")\n",
    "\n",
    "    detection_inicialitation_time = datetime.utcnow().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "\n",
    "    if load_from_checkpoint:\n",
    "        command = f'python src/detection/run_detector_batch.py \"{MODEL_PATH}\" \"{IMAGES_PATH_JSON}\" \"{OUTPUT_FILE_PATH}\" --recursive --threshold \"{threshold}\" --checkpoint_path \"{CHECKPOINT_PATH}\" --checkpoint_frequency \"{CHECKPOINT_FREQ}\" --resume_from_checkpoint \"{CHECKPOINT_PATH}\"'\n",
    "    else:\n",
    "        command = f'python src/detection/run_detector_batch.py \"{MODEL_PATH}\" \"{IMAGES_PATH_JSON}\" \"{OUTPUT_FILE_PATH}\" --recursive --threshold \"{threshold}\" --checkpoint_path \"{CHECKPOINT_PATH}\" --checkpoint_frequency \"{CHECKPOINT_FREQ}\"'\n",
    "    os.system(command)\n",
    "\n",
    "    with open(OUTPUT_FILE_PATH, \"r\") as file:\n",
    "        data = json.load(file)\n",
    "\n",
    "    for image in data[\"images\"]:\n",
    "        image[\"file\"] = image[\"file\"].replace(\n",
    "            DATASET_PATH + \"/\",\n",
    "            \"\",\n",
    "        )\n",
    "\n",
    "    info = {\n",
    "        \"detection_initial_time\": detection_inicialitation_time,\n",
    "        \"detection_completion_time\": data[\"info\"][\"detection_completion_time\"],\n",
    "        \"format_version\": data[\"info\"][\"format_version\"],\n",
    "        \"detector\": data[\"info\"][\"detector\"],\n",
    "        \"detector_threshold\": threshold,\n",
    "        \"detector_metadata\": data[\"info\"][\"detector_metadata\"],\n",
    "    }\n",
    "\n",
    "    final_output = {\n",
    "        \"images\": data[\"images\"],\n",
    "        \"detection_categories\": data[\"detection_categories\"],\n",
    "        \"info\": info,\n",
    "    }\n",
    "\n",
    "    threshold_str = str(threshold).replace(\".\", \"_\")\n",
    "    json_name = f\"{len(data['images'])}_images_{threshold_str}_threshold.json\"\n",
    "    model_name = os.path.basename(MODEL_PATH).split(\".\")[0]\n",
    "    tmp_path = os.path.join(OUTPUT_PATH, model_name)\n",
    "    os.makedirs(tmp_path, exist_ok=True)\n",
    "    NEW_OUTPUT_FILE_PATH = os.path.join(tmp_path, json_name)\n",
    "\n",
    "    with open(NEW_OUTPUT_FILE_PATH, \"w\") as f:\n",
    "        json.dump(final_output, f, indent=1)\n",
    "\n",
    "    os.remove(OUTPUT_FILE_PATH)\n",
    "\n",
    "    data = final_output\n",
    "\n",
    "    dataset = dt.load_from_csv(TRAIN_CSV)\n",
    "\n",
    "    report_columns = [\n",
    "        \"file_name\",\n",
    "        \"label\",\n",
    "        \"threshold\",\n",
    "        \"detector_label\",\n",
    "        \"false_positive\",\n",
    "        \"false_negative\",\n",
    "        \"time_inference\",\n",
    "    ]\n",
    "    report = pd.DataFrame(columns=report_columns)\n",
    "\n",
    "    model = data[\"info\"][\"detector\"].split(\".\")[0]\n",
    "\n",
    "    for image in data[\"images\"]:\n",
    "        image_file = image[\"file\"]\n",
    "        indexes = dataset[dataset[\"file_name\"] == image_file]\n",
    "\n",
    "        if len(indexes) == 1:\n",
    "            label = int(indexes[\"label\"].iloc[0])\n",
    "            detector_label = 1 if image[\"max_detection_conf\"] > 0.0 else 0\n",
    "            false_positive = int(label == 0 and detector_label == 1)\n",
    "            false_negative = int(label == 1 and detector_label == 0)\n",
    "            time_inference = image[\"time_inference\"]\n",
    "\n",
    "            new_row = {\n",
    "                \"file_name\": image_file,\n",
    "                \"label\": label,\n",
    "                \"threshold\": data[\"info\"][\"detector_threshold\"],\n",
    "                \"detector_label\": detector_label,\n",
    "                \"false_positive\": false_positive,\n",
    "                \"false_negative\": false_negative,\n",
    "                \"time_inference\": time_inference,\n",
    "            }\n",
    "            report = pd.concat([report, pd.DataFrame([new_row])], ignore_index=True)\n",
    "\n",
    "    report_name = f\"{len(data['images'])}_images_{model}.csv\"\n",
    "    REPORT_CSV_FILE = os.path.join(REPORT_CSV, report_name)\n",
    "\n",
    "    if os.path.exists(REPORT_CSV_FILE):\n",
    "        existing_report = dt.load_from_csv(REPORT_CSV_FILE)\n",
    "        new_report = pd.concat([existing_report, report], ignore_index=True)\n",
    "        dt.dataset_to_csv(new_report, REPORT_CSV_FILE)\n",
    "    else:\n",
    "        dt.dataset_to_csv(report, REPORT_CSV_FILE)\n",
    "\n",
    "    positivos_reales = report[\"label\"].sum()\n",
    "    positivos_cubiertos = report[\n",
    "        (report[\"label\"] == 1) & (report[\"detector_label\"] == 1)\n",
    "    ].shape[0]\n",
    "\n",
    "    positives_coberture = (\n",
    "        (positivos_cubiertos / positivos_reales) if positivos_reales > 0 else 0\n",
    "    )\n",
    "\n",
    "    load_from_checkpoint = False\n",
    "\n",
    "    print()\n",
    "    print(\"-----------------------------------------------------------------------\")\n",
    "    print(f\"El porcentaje de positivos cubiertos es {positives_coberture*100:.2f}%\")\n",
    "    print(\"-----------------------------------------------------------------------\")\n",
    "    print()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cameratraps-detector",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}