01_b_GenerateReducedDataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import src.data.Dataset as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RAW_DATASET: /Users/carlos/WORKSPACE/MegaClassifier/data/raw/28570Images_binary.csv\n"
     ]
    }
   ],
   "source": [
    "RAW_DATASET = os.path.abspath(\"./data/raw/28570Images_binary.csv\")\n",
    "\n",
    "print(f\"RAW_DATASET: {RAW_DATASET}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The file /Users/carlos/WORKSPACE/MegaClassifier/data/raw/28570Images_binary.csv has been successfully opened.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\\vacia\\empty_IslandConservationCameraTraps\\mic...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\\vacia\\NOANIMAL_ZOO_5_6_7_9\\19_20210618 (1118)...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\\vacia\\NOANIMAL_ZOO_5_6_7_9\\50_20210408 (9130)...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\\cervidae\\CERVIDREDORFALLOWDEER_ZOO_1_4\\21_202...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\\cervidae\\CERVIDREDORFALLOWDEER_ZOO_1_4\\2_2020...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28565</th>\n",
       "      <td>\\zorro\\zorro_PNC_2013-I_RED_ME\\686IM000106.JPG</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28566</th>\n",
       "      <td>\\vacia\\empty_SnapshotKaroo\\KAR_S1_C03_R1_IMAG0...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28567</th>\n",
       "      <td>\\vacia\\NOANIMAL_ZOO_5_6_7_9\\11_20210318 (114).JPG</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28568</th>\n",
       "      <td>\\meloncillo\\Meloncillo_PEPE_IREC_JC\\IMAG0270_(...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28569</th>\n",
       "      <td>\\lince\\linces_Linces2_Leo_JC\\2_Lince_Linces2_L...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>28570 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               file_name  label\n",
       "0      \\vacia\\empty_IslandConservationCameraTraps\\mic...      0\n",
       "1      \\vacia\\NOANIMAL_ZOO_5_6_7_9\\19_20210618 (1118)...      0\n",
       "2      \\vacia\\NOANIMAL_ZOO_5_6_7_9\\50_20210408 (9130)...      0\n",
       "3      \\cervidae\\CERVIDREDORFALLOWDEER_ZOO_1_4\\21_202...      1\n",
       "4      \\cervidae\\CERVIDREDORFALLOWDEER_ZOO_1_4\\2_2020...      1\n",
       "...                                                  ...    ...\n",
       "28565     \\zorro\\zorro_PNC_2013-I_RED_ME\\686IM000106.JPG      1\n",
       "28566  \\vacia\\empty_SnapshotKaroo\\KAR_S1_C03_R1_IMAG0...      0\n",
       "28567  \\vacia\\NOANIMAL_ZOO_5_6_7_9\\11_20210318 (114).JPG      0\n",
       "28568  \\meloncillo\\Meloncillo_PEPE_IREC_JC\\IMAG0270_(...      1\n",
       "28569  \\lince\\linces_Linces2_Leo_JC\\2_Lince_Linces2_L...      1\n",
       "\n",
       "[28570 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = dt.load_from_csv(RAW_DATASET)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples in raw dataset: 28570\n",
      "label\n",
      "0     9665\n",
      "1    18905\n",
      "Name: count, dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "number_raw_samples = len(dataset[\"file_name\"].values)\n",
    "number_raw_empty = len(dataset[dataset[\"label\"] == 0])\n",
    "number_raw_animal = len(dataset[dataset[\"label\"] == 1])\n",
    "\n",
    "samples_empty_class = number_raw_empty\n",
    "# samples_empty_class = 50\n",
    "\n",
    "print(\"Number of samples in raw dataset: \" + str(number_raw_samples))\n",
    "label_counts = dataset[\"label\"].value_counts()\n",
    "label_counts_sorted = label_counts.sort_index()\n",
    "print(label_counts_sorted)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset has been successfully saved to /Users/carlos/WORKSPACE/MegaClassifier/data/raw/28570Images_binary.csv\n",
      "Number of samples in set dataset: 28570\n",
      "label\n",
      "0     9665\n",
      "1    18905\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "dataset_crop = dt.crop_dataset(dataset, samples_empty_class)\n",
    "\n",
    "number_set_samples = len(dataset_crop[\"file_name\"].values)\n",
    "number_set_empty = len(dataset_crop[dataset_crop[\"label\"] == 0])\n",
    "number_set_animal = len(dataset_crop[dataset_crop[\"label\"] == 1])\n",
    "\n",
    "csv_crop_path = os.path.join(\n",
    "    RAW_DATASET[: -len(os.path.basename(RAW_DATASET))],\n",
    "    str(number_set_samples) + \"Images_binary.csv\",\n",
    ")\n",
    "dt.dataset_to_csv(dataset_crop, csv_crop_path)\n",
    "\n",
    "print(\"Number of samples in set dataset: \" + str(number_set_samples))\n",
    "label_counts = dataset_crop[\"label\"].value_counts()\n",
    "label_counts_sorted = label_counts.sort_index()\n",
    "print(label_counts_sorted)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cameratraps-detector",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}