dataloader_techniques.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ea26d47a-ab58-4ed2-97f7-a9d3164708e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "import langchain\n",
    "openai.api_key = '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'\n",
    "os.environ['OPENAI_API_KEY'] = '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'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ecbf40b0-a4b1-4104-8836-0b4221a38938",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import PyPDFLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7f7e68c1-75e1-430f-9850-b2f03765cb62",
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = PyPDFLoader(\"DBMS_MST.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bb021c9d-bfef-423c-abfc-e3e18a23354c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4fb75894-bd86-4abf-ab62-3e3dddcd3e58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f0524ff0-cedd-4db0-ad13-eb226722d2fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "page = pages[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d57e0790-b59c-41e1-9457-18e71db9bb8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Alternate key?  \n",
      " \n",
      "All the keys which are not primary key are called an alternate key. \n",
      "It is a candidate key which is currently not the primary key. \n",
      "However, A table may  have single or multiple choices for the primary key. \n",
      " \n",
      "Example: In this table, StudID, Roll No, Email are qualified to become a primary key. \n",
      "But since StudID is the primary key , Roll No, Email becomes the alternative key . \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(page.page_content[:500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "01855d58-70a7-4c0d-aed7-29999b387ff4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'source': 'DBMS_MST.pdf', 'page': 100}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page.metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5e0339a8-b1cc-499f-aab3-d74dabc93c05",
   "metadata": {},
   "outputs": [],
   "source": [
    "# youtube loader\n",
    "from langchain.document_loaders.generic import GenericLoader\n",
    "from langchain.document_loaders.parsers import OpenAIWhisperParser\n",
    "from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1d2e70af-2f17-46c2-bbee-01b7c0f2971f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[youtube] Extracting URL: https://www.youtube.com/watch?v=RBumgq5yVrA&ab_channel=Passenger\n",
      "[youtube] RBumgq5yVrA: Downloading webpage\n",
      "[youtube] RBumgq5yVrA: Downloading ios player API JSON\n",
      "[youtube] RBumgq5yVrA: Downloading android player API JSON\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: [youtube] Skipping player responses from android clients (got player responses for video \"aQvGIIdgFDM\" instead of \"RBumgq5yVrA\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[youtube] RBumgq5yVrA: Downloading player 9383995e\n",
      "[youtube] RBumgq5yVrA: Downloading m3u8 information\n",
      "[info] RBumgq5yVrA: Downloading 1 format(s): 140\n",
      "[download] Destination: youtube\\Passenger ｜ Let Her Go (Official Video).m4a\n",
      "[download] 100% of    3.92MiB in 00:00:01 at 3.53MiB/s   \n",
      "[FixupM4a] Correcting container of \"youtube\\Passenger ｜ Let Her Go (Official Video).m4a\"\n",
      "[ExtractAudio] Not converting audio youtube\\Passenger ｜ Let Her Go (Official Video).m4a; file is already in target format m4a\n",
      "Transcribing part 1!\n",
      "Transcribing part 1!\n"
     ]
    }
   ],
   "source": [
    "url = \"https://www.youtube.com/watch?v=RBumgq5yVrA&ab_channel=Passenger\"\n",
    "save_dir = \"youtube/\"\n",
    "loader = GenericLoader(\n",
    "    YoutubeAudioLoader([url],save_dir),\n",
    "    OpenAIWhisperParser()\n",
    ")\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4555ec49-adec-476c-bbf9-67935c63370e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "page_content='We are turning right to Session Road We are turning right to Session Road We are turning right to Session Road We are turning right to Session Road We are turning right to Session Road We are turning right to Session Road We are turning right to Session Road We are turning right to Session Road We are turning right to Session Road We are turning right to Session Road We are turning right to Session Road We are turning right to Session Road' metadata={'source': 'youtube\\\\Let Her Go x Husn (Gravero Mashup) ｜ Anuv Jain.m4a', 'chunk': 0}\n",
      "\n",
      "\n",
      "page_content=\"🎵Music🎵 🎵Well you only need the light when it's burning low🎵 🎵Only miss the sun when it starts to snow🎵 🎵Only know you love her when you let her go🎵 🎵Only know you've been high when you're feeling low🎵 🎵Only hate the road when you're missing home🎵 🎵Only know you love her when you let her go🎵 🎵And you let her go🎵 🎵Music🎵 🎵Staring at the bottom of your glass🎵 🎵Hoping one day you'll make a dream last🎵 🎵But dreams come slow and they go so fast🎵 🎵You see her when you close your eyes🎵 🎵Maybe one day you'll understand why🎵 🎵Everything you touch surely dies🎵 🎵But you only need the light when it's burning low🎵 🎵Only miss the sun when it starts to snow🎵 🎵Only know you love her when you let her go🎵 🎵Only know you've been high when you're feeling low🎵 🎵Only hate the road when you're missing home🎵 🎵Only know you love her when you let her go🎵 🎵Staring at the ceiling in the dark🎵 🎵Same old empty feeling in your heart🎵 🎵Cause love comes slow and it goes so fast🎵 🎵Well you see her when you fall asleep🎵 🎵But never to touch and never to keep cause🎵 🎵You loved her too much and you dive too deep🎵 🎵But you only need the light when it's burning low🎵 🎵Only miss the sun when it starts to snow🎵 🎵Only know you love her when you let her go🎵 🎵Only know you've been high when you're feeling low🎵 🎵Only hate the road when you're missing home🎵 🎵Only know you love her when you let her go🎵 🎵And you let her go🎵 🎵And you let her go🎵 🎵Well you let her go🎵 🎵And you let her go🎵 🎵Cause you only need the light when it's burning low🎵 🎵Only miss the sun when it starts to snow🎵 🎵Only know you love her when you let her go🎵 🎵Only know you've been high when you're feeling low🎵 🎵Only hate the road when you're missing home🎵 🎵Only know you love her when you let her go🎵 🎵Cause you only need the light when it's burning low🎵 🎵Only miss the sun when it starts to snow🎵 🎵Only know you love her when you let her go🎵 🎵Only know you've been high when you're feeling low🎵 🎵Only hate the road when you're missing home🎵 🎵Only know you love her when you let her go🎵 🎵And you let her go🎵\" metadata={'source': 'youtube\\\\Passenger ｜ Let Her Go (Official Video).m4a', 'chunk': 0}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in docs:\n",
    "    print(i)\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "bdea9cd7-6cef-421b-a719-00713fee9db6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#URL document loader:\n",
    "from langchain.document_loaders import WebBaseLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8380b7b0-319b-46c6-a05b-4f3ba2e1bc76",
   "metadata": {},
   "outputs": [],
   "source": [
    "web_loader = WebBaseLoader('https://timesofindia.indiatimes.com/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "aa7fe9a9-146b-4445-b32e-4306ffccd7ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = web_loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67875862-cb43-4ce4-8079-2b3800be7f55",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "langchain",
   "language": "python",
   "name": "langchain"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}