SpeakerID/denoise.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T15:04:48.482686400Z",
     "start_time": "2023-06-13T15:04:47.953609600Z"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-13 23:06:48,248 - modelscope - INFO - initiate model from speech_frcrn_ans_cirm_16k\n",
      "2023-06-13 23:06:48,252 - modelscope - INFO - initiate model from location speech_frcrn_ans_cirm_16k.\n",
      "2023-06-13 23:06:48,268 - modelscope - INFO - initialize model from speech_frcrn_ans_cirm_16k\n",
      "2023-06-13 23:06:49,100 - modelscope - WARNING - No preprocessor field found in cfg.\n",
      "2023-06-13 23:06:49,101 - modelscope - WARNING - No val key and type key found in preprocessor domain of configuration.json file.\n",
      "2023-06-13 23:06:49,102 - modelscope - WARNING - Cannot find available config to build preprocessor at mode inference, current config: {'model_dir': 'speech_frcrn_ans_cirm_16k'}. trying to build by task and model information.\n",
      "2023-06-13 23:06:49,103 - modelscope - WARNING - No preprocessor key ('speech_frcrn_ans_cirm_16k', 'acoustic-noise-suppression') found in PREPROCESSOR_MAP, skip building preprocessor.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001B[32m2023-06-13 23:06:49\u001B[0m | \u001B[1mINFO    \u001B[0m | \u001B[36mDF\u001B[0m | \u001B[1mLoading model settings of DeepFilterNet3\u001B[0m\n",
      "\u001B[32m2023-06-13 23:06:49\u001B[0m | \u001B[1mINFO    \u001B[0m | \u001B[36mDF\u001B[0m | \u001B[1mInitializing model `deepfilternet3`\u001B[0m\n",
      "\u001B[32m2023-06-13 23:06:49\u001B[0m | \u001B[1mINFO    \u001B[0m | \u001B[36mDF\u001B[0m | \u001B[1mFound checkpoint DeepFilterNet3/checkpoints/model_120.ckpt.best with epoch 120\u001B[0m\n",
      "\u001B[32m2023-06-13 23:06:49\u001B[0m | \u001B[1mINFO    \u001B[0m | \u001B[36mDF\u001B[0m | \u001B[1mRunning on device cuda:0\u001B[0m\n",
      "\u001B[32m2023-06-13 23:06:49\u001B[0m | \u001B[1mINFO    \u001B[0m | \u001B[36mDF\u001B[0m | \u001B[1mModel loaded\u001B[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs:(1, 240000)\n",
      "padding: 24000\n",
      "inputs after padding:(1, 264000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/venvs/til2023/lib/python3.8/site-packages/torchaudio/functional/functional.py:1458: UserWarning: \"sinc_interpolation\" resampling method name is being deprecated and replaced by \"sinc_interp_hann\" in the next release. The default behavior remains unchanged.\n",
      "  warnings.warn(\n",
      " 20%|██        | 1/5 [00:01<00:05,  1.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs:(1, 240000)\n",
      "padding: 24000\n",
      "inputs after padding:(1, 264000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/venvs/til2023/lib/python3.8/site-packages/torchaudio/functional/functional.py:1458: UserWarning: \"sinc_interpolation\" resampling method name is being deprecated and replaced by \"sinc_interp_hann\" in the next release. The default behavior remains unchanged.\n",
      "  warnings.warn(\n",
      " 40%|████      | 2/5 [00:02<00:03,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs:(1, 240000)\n",
      "padding: 24000\n",
      "inputs after padding:(1, 264000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/venvs/til2023/lib/python3.8/site-packages/torchaudio/functional/functional.py:1458: UserWarning: \"sinc_interpolation\" resampling method name is being deprecated and replaced by \"sinc_interp_hann\" in the next release. The default behavior remains unchanged.\n",
      "  warnings.warn(\n",
      " 60%|██████    | 3/5 [00:02<00:01,  1.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs:(1, 240000)\n",
      "padding: 24000\n",
      "inputs after padding:(1, 264000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/venvs/til2023/lib/python3.8/site-packages/torchaudio/functional/functional.py:1458: UserWarning: \"sinc_interpolation\" resampling method name is being deprecated and replaced by \"sinc_interp_hann\" in the next release. The default behavior remains unchanged.\n",
      "  warnings.warn(\n",
      " 80%|████████  | 4/5 [00:03<00:00,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs:(1, 240000)\n",
      "padding: 24000\n",
      "inputs after padding:(1, 264000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/venvs/til2023/lib/python3.8/site-packages/pyloudnorm/normalize.py:62: UserWarning: Possible clipped samples in output.\n",
      "  warnings.warn(\"Possible clipped samples in output.\")\n",
      "100%|██████████| 5/5 [00:04<00:00,  1.17it/s]\n"
     ]
    }
   ],
   "source": [
    "from df.enhance import enhance, init_df, load_audio, save_audio\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm\n",
    "import glob\n",
    "import os\n",
    "from modelscope.pipelines import pipeline\n",
    "from modelscope.utils.constant import Tasks\n",
    "import torch\n",
    "import soundfile as sf\n",
    "import pyloudnorm as pyln\n",
    "import librosa\n",
    "\n",
    "\n",
    "OUTPUT_DIR = Path(\"val_denoised\")\n",
    "\n",
    "# make the OUTPUT_DIR if it doesn't exist\n",
    "os.makedirs(OUTPUT_DIR, exist_ok=True)\n",
    "frcrn = pipeline(Tasks.acoustic_noise_suppression, model='speech_frcrn_ans_cirm_16k')\n",
    "# Load default model\n",
    "model, df_state, model_name_suffix = init_df(model_base_dir=\"DeepFilterNet3/\")\n",
    "# Get our SpeakerID audio\n",
    "speakerID_audio_folder = \"val\"\n",
    "audio_paths = glob.glob(f\"{speakerID_audio_folder}/*.wav\")\n",
    "# Enhance each audio\n",
    "for audio_path in tqdm(audio_paths):\n",
    "    # get the audio filename without .wav extension\n",
    "    audio_filename = os.path.splitext(os.path.basename(audio_path))[0]\n",
    "    output_path = OUTPUT_DIR/f\"{audio_filename}_enhanced.wav\"\n",
    "    frcrn(audio_path, output_path=output_path)\n",
    "\n",
    "    data, rate = sf.read(output_path) # load audio\n",
    "    # peak normalize audio to -0.1 dB as frcrn tend to output very soft\n",
    "    peak_normalized_audio = pyln.normalize.peak(data, -0.1)  # not using loudness norm here as it causes a bit of clipping\n",
    "    sf.write(output_path, peak_normalized_audio, rate)\n",
    "    if 'PALMTREE' not in audio_path:\n",
    "        audio, _ = load_audio(output_path, sr=df_state.sr())\n",
    "        # Denoise the audio\n",
    "        enhanced = enhance(model, df_state, audio)\n",
    "        # Save for listening\n",
    "        save_audio(output_path, enhanced, df_state.sr(), dtype=torch.float16)  # default is torch.int16 which causes clipping on some audios\n",
    "    audio, sr = librosa.load(output_path, sr=16000)  # downsample to 16Khz\n",
    "    sf.write(output_path, audio, sr)  # save the downsampled one\n",
    "\n",
    "    data, rate = sf.read(output_path) # load audio\n",
    "    if 'PALMTREE' not in audio_path:\n",
    "        # peak normalize audio to -0.1 dB as frcrn tend to output very soft\n",
    "        normalized_audio = pyln.normalize.peak(data, -0.1)  # not using loudness norm here as it causes a bit of clipping on non palmtree clips\n",
    "    else:  # PALMTREE ones need louder\n",
    "        # measure the loudness first \n",
    "        meter = pyln.Meter(rate) # create BS.1770 meter\n",
    "        loudness = meter.integrated_loudness(data)\n",
    "        # loudness normalize audio to -18 dB LUFS\n",
    "        normalized_audio = pyln.normalize.loudness(data, loudness, -18.0)\n",
    "    sf.write(output_path, normalized_audio, rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 17:20:54,980 - modelscope - INFO - initiate model from speech_frcrn_ans_cirm_16k\n",
      "2023-06-14 17:20:54,985 - modelscope - INFO - initiate model from location speech_frcrn_ans_cirm_16k.\n",
      "2023-06-14 17:20:55,001 - modelscope - INFO - initialize model from speech_frcrn_ans_cirm_16k\n",
      "2023-06-14 17:20:55,895 - modelscope - WARNING - No preprocessor field found in cfg.\n",
      "2023-06-14 17:20:55,896 - modelscope - WARNING - No val key and type key found in preprocessor domain of configuration.json file.\n",
      "2023-06-14 17:20:55,897 - modelscope - WARNING - Cannot find available config to build preprocessor at mode inference, current config: {'model_dir': 'speech_frcrn_ans_cirm_16k'}. trying to build by task and model information.\n",
      "2023-06-14 17:20:55,897 - modelscope - WARNING - No preprocessor key ('speech_frcrn_ans_cirm_16k', 'acoustic-noise-suppression') found in PREPROCESSOR_MAP, skip building preprocessor.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001B[32m2023-06-14 17:20:55\u001B[0m | \u001B[1mINFO    \u001B[0m | \u001B[36mDF\u001B[0m | \u001B[1mLoading model settings of DeepFilterNet3\u001B[0m\n",
      "\u001B[32m2023-06-14 17:20:55\u001B[0m | \u001B[1mINFO    \u001B[0m | \u001B[36mDF\u001B[0m | \u001B[1mInitializing model `deepfilternet3`\u001B[0m\n",
      "\u001B[32m2023-06-14 17:20:55\u001B[0m | \u001B[1mINFO    \u001B[0m | \u001B[36mDF\u001B[0m | \u001B[1mFound checkpoint DeepFilterNet3/checkpoints/model_120.ckpt.best with epoch 120\u001B[0m\n",
      "\u001B[32m2023-06-14 17:20:56\u001B[0m | \u001B[1mINFO    \u001B[0m | \u001B[36mDF\u001B[0m | \u001B[1mRunning on device cuda:0\u001B[0m\n",
      "\u001B[32m2023-06-14 17:20:56\u001B[0m | \u001B[1mINFO    \u001B[0m | \u001B[36mDF\u001B[0m | \u001B[1mModel loaded\u001B[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs:(1, 240000)\n",
      "padding: 24000\n",
      "inputs after padding:(1, 264000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/venvs/til2023/lib/python3.8/site-packages/torchaudio/functional/functional.py:1458: UserWarning: \"sinc_interpolation\" resampling method name is being deprecated and replaced by \"sinc_interp_hann\" in the next release. The default behavior remains unchanged.\n",
      "  warnings.warn(\n",
      " 20%|██        | 1/5 [00:01<00:04,  1.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-45.57793667402997\n",
      "inputs:(1, 240000)\n",
      "padding: 24000\n",
      "inputs after padding:(1, 264000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/venvs/til2023/lib/python3.8/site-packages/torchaudio/functional/functional.py:1458: UserWarning: \"sinc_interpolation\" resampling method name is being deprecated and replaced by \"sinc_interp_hann\" in the next release. The default behavior remains unchanged.\n",
      "  warnings.warn(\n",
      " 40%|████      | 2/5 [00:02<00:03,  1.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-45.46863014104234\n",
      "inputs:(1, 240000)\n",
      "padding: 24000\n",
      "inputs after padding:(1, 264000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/venvs/til2023/lib/python3.8/site-packages/torchaudio/functional/functional.py:1458: UserWarning: \"sinc_interpolation\" resampling method name is being deprecated and replaced by \"sinc_interp_hann\" in the next release. The default behavior remains unchanged.\n",
      "  warnings.warn(\n",
      " 60%|██████    | 3/5 [00:02<00:01,  1.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-53.35755958759073\n",
      "inputs:(1, 240000)\n",
      "padding: 24000\n",
      "inputs after padding:(1, 264000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/venvs/til2023/lib/python3.8/site-packages/torchaudio/functional/functional.py:1458: UserWarning: \"sinc_interpolation\" resampling method name is being deprecated and replaced by \"sinc_interp_hann\" in the next release. The default behavior remains unchanged.\n",
      "  warnings.warn(\n",
      " 80%|████████  | 4/5 [00:03<00:00,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-40.452659533725395\n",
      "inputs:(1, 240000)\n",
      "padding: 24000\n",
      "inputs after padding:(1, 264000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/venvs/til2023/lib/python3.8/site-packages/torchaudio/functional/functional.py:1458: UserWarning: \"sinc_interpolation\" resampling method name is being deprecated and replaced by \"sinc_interp_hann\" in the next release. The default behavior remains unchanged.\n",
      "  warnings.warn(\n",
      "100%|██████████| 5/5 [00:04<00:00,  1.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-64.21088891293473\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from df.enhance import enhance, init_df, load_audio, save_audio\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm\n",
    "import glob\n",
    "import os\n",
    "from modelscope.pipelines import pipeline\n",
    "from modelscope.utils.constant import Tasks\n",
    "import torch\n",
    "import soundfile as sf\n",
    "import pyloudnorm as pyln\n",
    "import librosa\n",
    "\n",
    "\n",
    "OUTPUT_DIR = Path(\"denoised_test\")\n",
    "\n",
    "# make the OUTPUT_DIR if it doesn't exist\n",
    "os.makedirs(OUTPUT_DIR, exist_ok=True)\n",
    "frcrn = pipeline(Tasks.acoustic_noise_suppression, model='speech_frcrn_ans_cirm_16k')\n",
    "# Load default model\n",
    "model, df_state, model_name_suffix = init_df(model_base_dir=\"DeepFilterNet3/\")\n",
    "# Get our SpeakerID audio\n",
    "speakerID_audio_folder = \"test\"\n",
    "audio_paths = glob.glob(f\"{speakerID_audio_folder}/*.wav\")\n",
    "# Enhance each audio\n",
    "for audio_path in tqdm(audio_paths):\n",
    "    # get the audio filename without .wav extension\n",
    "    audio_filename = os.path.splitext(os.path.basename(audio_path))[0]\n",
    "    output_path = OUTPUT_DIR/f\"{audio_filename}_enhanced.wav\"\n",
    "    frcrn(audio_path, output_path=output_path)\n",
    "\n",
    "    data, rate = sf.read(output_path) # load audio\n",
    "    # peak normalize audio to -0.1 dB as frcrn tend to output very soft\n",
    "    peak_normalized_audio = pyln.normalize.peak(data, -0.1)  # not using loudness norm here as it causes a bit of clipping\n",
    "    sf.write(output_path, peak_normalized_audio, rate)\n",
    "    audio, _ = load_audio(output_path, sr=df_state.sr())\n",
    "    # Denoise the audio\n",
    "    enhanced = enhance(model, df_state, audio)\n",
    "    # Save for listening\n",
    "    save_audio(output_path, enhanced, df_state.sr(), dtype=torch.float16)  # default is torch.int16 which causes clipping on some audios\n",
    "    audio, sr = librosa.load(output_path, sr=16000)  # downsample to 16Khz\n",
    "    sf.write(output_path, audio, sr)  # save the downsampled one\n",
    "\n",
    "    # data, rate = sf.read(output_path) # load audio\n",
    "    # if 'PALMTREE' not in audio_path:\n",
    "    #     # peak normalize audio to -0.1 dB as frcrn tend to output very soft\n",
    "    #     normalized_audio = pyln.normalize.peak(data, -0.1)  # not using loudness norm here as it causes a bit of clipping on non palmtree clips\n",
    "    # else:  # PALMTREE ones need louder\n",
    "    #     # measure the loudness first\n",
    "    #     meter = pyln.Meter(rate) # create BS.1770 meter\n",
    "    #     loudness = meter.integrated_loudness(data)\n",
    "    #     # loudness normalize audio to -18 dB LUFS\n",
    "    #     normalized_audio = pyln.normalize.loudness(data, loudness, -18.0)\n",
    "    # sf.write(output_path, normalized_audio, rate)\n",
    "    loudness_meter = pyln.Meter(rate)\n",
    "    data, rate = sf.read(output_path)  # load audio\n",
    "    loudness = loudness_meter.integrated_loudness(data)\n",
    "    print(loudness)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}