MCQ_Generation.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aweZgxXBDsOQ",
        "outputId": "9a4693c4-1cfe-485a-a3c1-bc2bea6287cf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.30.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.16.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.13.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.3.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.7.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.5.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (0.1.99)\n",
            "/usr/lib/python3.10/runpy.py:126: RuntimeWarning: 'nltk.downloader' found in sys.modules after import of package 'nltk', but prior to execution of 'nltk.downloader'; this may result in unpredictable behaviour\n",
            "  warn(RuntimeWarning(msg))\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "Requirement already satisfied: langdetect in /usr/local/lib/python3.10/dist-packages (1.0.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from langdetect) (1.16.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install -U transformers\n",
        "!pip install sentencepiece\n",
        "!python -m nltk.downloader punkt\n",
        "!pip install langdetect"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "rk3X75xXRH5r"
      },
      "outputs": [],
      "source": [
        "text1 = \"Gravity (from Latin gravitas, meaning 'weight'), or gravitation, is a natural phenomenon by which all \\\n",
        "things with mass or energy—including planets, stars, galaxies, and even light—are brought toward (or gravitate toward) \\\n",
        "one another. On Earth, gravity gives weight to physical objects, and the Moon's gravity causes the ocean tides. \\\n",
        "The gravitational attraction of the original gaseous matter present in the Universe caused it to begin coalescing \\\n",
        "and forming stars and caused the stars to group together into galaxies, so gravity is responsible for many of \\\n",
        "the large-scale structures in the Universe. Gravity has an infinite range, although its effects become increasingly \\\n",
        "weaker as objects get further away\"\n",
        "\n",
        "text2 = \"Je m’appelle Jessica. Je suis une fille, je suis française et j’ai treize ans.\\\n",
        " Je vais à l’école à Nice, mais j’habite à Cagnes-Sur-Mer. J’ai deux frères. Le premier s’appelle Thomas, \\\n",
        " il a quatorze ans. Le second s’appelle Yann et il a neuf ans. Mon papa est italien et il est fleuriste. \\\n",
        " Ma mère est allemande et est avocate. Mes frères et moi parlons français, italien et allemand à la maison.\\\n",
        "  Nous avons une grande maison avec un chien, un poisson et deux chats.\"\n",
        "\n",
        "text3 = \"सील से भरी हुई यह छोटी सी कोठरी, जिसकी दीवारों से गरीबी असहाय के असमर्थ साथी की तरह चिपटी हुई है।\\\n",
        "छत इतनी टूटी-फूटी जैसे गिरने ही वाली हो। किसी जमाने में इसमें एक लैंप लटक रहा था, जिसका कंकाल मात्र आज भी टंगा हुआ है।\"\n",
        "\n",
        "text4 = \"모든 사람은 교육을 받을 권리를 가진다 . 교육은 최소한 초등 및 기초단계에서는 무상이어야 한다. 초등교육은 의무적이어야 한다. 기술 및 직업교육은 일반적으로 접근이 가능하여야 하며, 고등교육은 모든 사람에게 실력에 근거하여 동등하게 접근 가능하여야 한다.교육은 인격의 완전한 발전과 인권과 기본적\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MvBFbCVGELuW"
      },
      "source": [
        "## Single task QA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0Xnay1hqO0P0",
        "outputId": "df7a1b7a-62c0-44fb-83df-6007bd5162a8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "1/1 [==============================] - 2s 2s/step - loss: 3.6874 - accuracy: 0.0741\n",
            "Epoch 2/50\n",
            "1/1 [==============================] - 0s 66ms/step - loss: 3.6710 - accuracy: 0.2963\n",
            "Epoch 3/50\n",
            "1/1 [==============================] - 0s 63ms/step - loss: 3.6535 - accuracy: 0.2407\n",
            "Epoch 4/50\n",
            "1/1 [==============================] - 0s 67ms/step - loss: 3.6325 - accuracy: 0.2407\n",
            "Epoch 5/50\n",
            "1/1 [==============================] - 0s 66ms/step - loss: 3.6044 - accuracy: 0.2222\n",
            "Epoch 6/50\n",
            "1/1 [==============================] - 0s 66ms/step - loss: 3.5622 - accuracy: 0.2222\n",
            "Epoch 7/50\n",
            "1/1 [==============================] - 0s 65ms/step - loss: 3.4914 - accuracy: 0.2222\n",
            "Epoch 8/50\n",
            "1/1 [==============================] - 0s 65ms/step - loss: 3.3610 - accuracy: 0.2222\n",
            "Epoch 9/50\n",
            "1/1 [==============================] - 0s 74ms/step - loss: 3.1487 - accuracy: 0.2222\n",
            "Epoch 10/50\n",
            "1/1 [==============================] - 0s 65ms/step - loss: 3.0279 - accuracy: 0.2222\n",
            "Epoch 11/50\n",
            "1/1 [==============================] - 0s 65ms/step - loss: 3.0911 - accuracy: 0.2222\n",
            "Epoch 12/50\n",
            "1/1 [==============================] - 0s 75ms/step - loss: 2.9481 - accuracy: 0.2222\n",
            "Epoch 13/50\n",
            "1/1 [==============================] - 0s 67ms/step - loss: 2.9159 - accuracy: 0.2407\n",
            "Epoch 14/50\n",
            "1/1 [==============================] - 0s 67ms/step - loss: 2.9203 - accuracy: 0.3333\n",
            "Epoch 15/50\n",
            "1/1 [==============================] - 0s 67ms/step - loss: 2.9250 - accuracy: 0.2963\n",
            "Epoch 16/50\n",
            "1/1 [==============================] - 0s 69ms/step - loss: 2.9254 - accuracy: 0.2963\n",
            "Epoch 17/50\n",
            "1/1 [==============================] - 0s 67ms/step - loss: 2.9198 - accuracy: 0.3148\n",
            "Epoch 18/50\n",
            "1/1 [==============================] - 0s 68ms/step - loss: 2.9088 - accuracy: 0.3148\n",
            "Epoch 19/50\n",
            "1/1 [==============================] - 0s 71ms/step - loss: 2.8945 - accuracy: 0.3148\n",
            "Epoch 20/50\n",
            "1/1 [==============================] - 0s 66ms/step - loss: 2.8787 - accuracy: 0.2778\n",
            "Epoch 21/50\n",
            "1/1 [==============================] - 0s 64ms/step - loss: 2.8611 - accuracy: 0.2963\n",
            "Epoch 22/50\n",
            "1/1 [==============================] - 0s 64ms/step - loss: 2.8407 - accuracy: 0.2963\n",
            "Epoch 23/50\n",
            "1/1 [==============================] - 0s 72ms/step - loss: 2.8171 - accuracy: 0.2963\n",
            "Epoch 24/50\n",
            "1/1 [==============================] - 0s 68ms/step - loss: 2.7916 - accuracy: 0.3333\n",
            "Epoch 25/50\n",
            "1/1 [==============================] - 0s 69ms/step - loss: 2.7659 - accuracy: 0.3148\n",
            "Epoch 26/50\n",
            "1/1 [==============================] - 0s 71ms/step - loss: 2.7409 - accuracy: 0.2963\n",
            "Epoch 27/50\n",
            "1/1 [==============================] - 0s 71ms/step - loss: 2.7152 - accuracy: 0.2963\n",
            "Epoch 28/50\n",
            "1/1 [==============================] - 0s 66ms/step - loss: 2.6863 - accuracy: 0.2963\n",
            "Epoch 29/50\n",
            "1/1 [==============================] - 0s 72ms/step - loss: 2.6533 - accuracy: 0.2963\n",
            "Epoch 30/50\n",
            "1/1 [==============================] - 0s 68ms/step - loss: 2.6176 - accuracy: 0.2778\n",
            "Epoch 31/50\n",
            "1/1 [==============================] - 0s 65ms/step - loss: 2.5803 - accuracy: 0.2778\n",
            "Epoch 32/50\n",
            "1/1 [==============================] - 0s 66ms/step - loss: 2.5405 - accuracy: 0.2778\n",
            "Epoch 33/50\n",
            "1/1 [==============================] - 0s 67ms/step - loss: 2.5001 - accuracy: 0.2963\n",
            "Epoch 34/50\n",
            "1/1 [==============================] - 0s 67ms/step - loss: 2.4644 - accuracy: 0.2963\n",
            "Epoch 35/50\n",
            "1/1 [==============================] - 0s 65ms/step - loss: 2.4324 - accuracy: 0.2963\n",
            "Epoch 36/50\n",
            "1/1 [==============================] - 0s 67ms/step - loss: 2.4036 - accuracy: 0.2963\n",
            "Epoch 37/50\n",
            "1/1 [==============================] - 0s 72ms/step - loss: 2.3820 - accuracy: 0.3148\n",
            "Epoch 38/50\n",
            "1/1 [==============================] - 0s 68ms/step - loss: 2.3635 - accuracy: 0.2963\n",
            "Epoch 39/50\n",
            "1/1 [==============================] - 0s 65ms/step - loss: 2.3489 - accuracy: 0.2963\n",
            "Epoch 40/50\n",
            "1/1 [==============================] - 0s 67ms/step - loss: 2.3343 - accuracy: 0.2963\n",
            "Epoch 41/50\n",
            "1/1 [==============================] - 0s 74ms/step - loss: 2.3175 - accuracy: 0.2963\n",
            "Epoch 42/50\n",
            "1/1 [==============================] - 0s 67ms/step - loss: 2.2970 - accuracy: 0.3148\n",
            "Epoch 43/50\n",
            "1/1 [==============================] - 0s 66ms/step - loss: 2.2732 - accuracy: 0.3148\n",
            "Epoch 44/50\n",
            "1/1 [==============================] - 0s 71ms/step - loss: 2.2456 - accuracy: 0.3148\n",
            "Epoch 45/50\n",
            "1/1 [==============================] - 0s 66ms/step - loss: 2.2192 - accuracy: 0.3333\n",
            "Epoch 46/50\n",
            "1/1 [==============================] - 0s 66ms/step - loss: 2.1925 - accuracy: 0.3333\n",
            "Epoch 47/50\n",
            "1/1 [==============================] - 0s 68ms/step - loss: 2.1654 - accuracy: 0.3148\n",
            "Epoch 48/50\n",
            "1/1 [==============================] - 0s 70ms/step - loss: 2.1364 - accuracy: 0.3333\n",
            "Epoch 49/50\n",
            "1/1 [==============================] - 0s 68ms/step - loss: 2.1059 - accuracy: 0.3704\n",
            "Epoch 50/50\n",
            "1/1 [==============================] - 0s 69ms/step - loss: 2.0823 - accuracy: 0.3333\n",
            "1/1 [==============================] - 0s 402ms/step\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, LSTM, Embedding\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "import re\n",
        "\n",
        "# Sample dataset in Hindi\n",
        "paragraphs = [\n",
        "  \"सील से भरी हुई यह छोटी सी कोठरी, जिसकी दीवारों से गरीबी असहाय के असमर्थ साथी की तरह चिपटी हुई है।\\\n",
        "छत इतनी टूटी-फूटी जैसे गिरने ही वाली हो। किसी जमाने में इसमें एक लैंप लटक रहा था, जिसका कंकाल मात्र आज भी टंगा हुआ है।\"\n",
        "]\n",
        "\n",
        "# Function to clean a Hindi sentence\n",
        "def clean_hindi_sentence(sentence):\n",
        "    sentence = re.sub(r'[^\\w\\s]', '', sentence) # Remove punctuations\n",
        "    sentence = re.sub(r'[a-zA-Z]', '', sentence) # Remove English characters\n",
        "    sentence = re.sub(r'\\d+', '', sentence) # Remove digits\n",
        "    sentence = sentence.strip() # Remove leading/trailing whitespaces\n",
        "    return sentence\n",
        "\n",
        "# Function to convert a paragraph to a list of sentences\n",
        "def paragraph_to_sentences(paragraph):\n",
        "    return paragraph.split('। ')\n",
        "\n",
        "# Convert the paragraphs to a list of sentences\n",
        "sentences = []\n",
        "for paragraph in paragraphs:\n",
        "    sentences.extend(paragraph_to_sentences(paragraph))\n",
        "\n",
        "# Clean the sentences\n",
        "cleaned_sentences = [clean_hindi_sentence(sentence) for sentence in sentences]\n",
        "\n",
        "# Tokenize the sentences\n",
        "word2idx = {\"<PAD>\": 0, \"<UNK>\": 1}\n",
        "idx2word = {0: \"<PAD>\", 1: \"<UNK>\"}\n",
        "for sentence in cleaned_sentences:\n",
        "    for word in sentence.split():\n",
        "        if word not in word2idx:\n",
        "            word2idx[word] = len(word2idx)\n",
        "            idx2word[len(idx2word)] = word\n",
        "\n",
        "# Convert the sentences to sequences of word indices\n",
        "sequences = []\n",
        "for sentence in cleaned_sentences:\n",
        "    sequence = []\n",
        "    for word in sentence.split():\n",
        "        if word in word2idx:\n",
        "            sequence.append(word2idx[word])\n",
        "        else:\n",
        "            sequence.append(word2idx[\"<UNK>\"])\n",
        "    sequences.append(sequence)\n",
        "\n",
        "# Pad the sequences to a fixed length\n",
        "max_len = max([len(sequence) for sequence in sequences])\n",
        "padded_sequences = pad_sequences(sequences, maxlen=max_len, padding=\"post\", value=word2idx[\"<PAD>\"])\n",
        "\n",
        "# Generate input and output data\n",
        "X = padded_sequences[:, :-1]\n",
        "y = padded_sequences[:, 1:]\n",
        "\n",
        "y = to_categorical(y, num_classes=len(word2idx))\n",
        "model = Sequential()\n",
        "model.add(Embedding(input_dim=len(word2idx), output_dim=100, input_length=max_len-1))\n",
        "model.add(LSTM(256, return_sequences=True))\n",
        "model.add(Dense(len(word2idx), activation=\"softmax\"))\n",
        "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
        "model.fit(X, y, batch_size=32, epochs=50)\n",
        "seed_text = \"एक लैंप जिसका कंकाल मात्र आज भी टंगा हुआ है।\"\n",
        "next_words = 10\n",
        "for _ in range(next_words):\n",
        "# Convert the seed text to a sequence of word indices\n",
        "  seed_sequence = []\n",
        "  for word in seed_text.split():\n",
        "    if word in word2idx:\n",
        "      seed_sequence.append(word2idx[word])\n",
        "    else:\n",
        "      seed_sequence.append(word2idx[\"<UNK>\"])\n",
        "# Pad the sequence to a fixed length\n",
        "    seed_padded = pad_sequences([seed_sequence], maxlen=max_len-1, padding=\"post\", value=word2idx[\"<PAD>\"])\n",
        "\n",
        "# Predict the next word\n",
        "predicted = model.predict(seed_padded)[0]\n",
        "predicted_idx = np.argmax(predicted)\n",
        "\n",
        "# Convert the predicted index to a word\n",
        "# Convert the predicted index to a word\n",
        "if predicted_idx in idx2word:\n",
        "    predicted_word = idx2word[predicted_idx]\n",
        "else:\n",
        "    predicted_word = \"<UNK>\"\n",
        "\n",
        "# Update the seed text\n",
        "seed_text += \" \" + predicted_word\n",
        "\n",
        "\n",
        "# Update the seed text\n",
        "seed_text += \" \" + predicted_word\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ljOTIfGTTaGR",
        "outputId": "4de218c2-1e45-4b73-d5fd-072f0879422b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "1/1 [==============================] - 2s 2s/step - loss: 2.9632 - accuracy: 0.0000e+00\n",
            "Epoch 2/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 2.9609 - accuracy: 0.0238\n",
            "Epoch 3/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.9586 - accuracy: 0.0714\n",
            "Epoch 4/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.9563 - accuracy: 0.1429\n",
            "Epoch 5/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.9539 - accuracy: 0.1429\n",
            "Epoch 6/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.9514 - accuracy: 0.1429\n",
            "Epoch 7/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.9489 - accuracy: 0.1667\n",
            "Epoch 8/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.9463 - accuracy: 0.1667\n",
            "Epoch 9/100\n",
            "1/1 [==============================] - 0s 14ms/step - loss: 2.9435 - accuracy: 0.1667\n",
            "Epoch 10/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.9406 - accuracy: 0.1905\n",
            "Epoch 11/100\n",
            "1/1 [==============================] - 0s 24ms/step - loss: 2.9376 - accuracy: 0.1905\n",
            "Epoch 12/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.9343 - accuracy: 0.1905\n",
            "Epoch 13/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.9308 - accuracy: 0.1905\n",
            "Epoch 14/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.9271 - accuracy: 0.1905\n",
            "Epoch 15/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.9230 - accuracy: 0.1905\n",
            "Epoch 16/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.9186 - accuracy: 0.1905\n",
            "Epoch 17/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.9138 - accuracy: 0.1905\n",
            "Epoch 18/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.9085 - accuracy: 0.1667\n",
            "Epoch 19/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.9027 - accuracy: 0.1667\n",
            "Epoch 20/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.8963 - accuracy: 0.1667\n",
            "Epoch 21/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.8892 - accuracy: 0.1667\n",
            "Epoch 22/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.8812 - accuracy: 0.1429\n",
            "Epoch 23/100\n",
            "1/1 [==============================] - 0s 25ms/step - loss: 2.8723 - accuracy: 0.1429\n",
            "Epoch 24/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.8623 - accuracy: 0.1429\n",
            "Epoch 25/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.8510 - accuracy: 0.1429\n",
            "Epoch 26/100\n",
            "1/1 [==============================] - 0s 24ms/step - loss: 2.8382 - accuracy: 0.1429\n",
            "Epoch 27/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.8238 - accuracy: 0.1429\n",
            "Epoch 28/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.8077 - accuracy: 0.1429\n",
            "Epoch 29/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.7898 - accuracy: 0.1429\n",
            "Epoch 30/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.7702 - accuracy: 0.1190\n",
            "Epoch 31/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.7494 - accuracy: 0.1190\n",
            "Epoch 32/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.7280 - accuracy: 0.1190\n",
            "Epoch 33/100\n",
            "1/1 [==============================] - 0s 28ms/step - loss: 2.7066 - accuracy: 0.1429\n",
            "Epoch 34/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.6858 - accuracy: 0.1429\n",
            "Epoch 35/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.6660 - accuracy: 0.1429\n",
            "Epoch 36/100\n",
            "1/1 [==============================] - 0s 24ms/step - loss: 2.6473 - accuracy: 0.1667\n",
            "Epoch 37/100\n",
            "1/1 [==============================] - 0s 23ms/step - loss: 2.6298 - accuracy: 0.1429\n",
            "Epoch 38/100\n",
            "1/1 [==============================] - 0s 24ms/step - loss: 2.6132 - accuracy: 0.1429\n",
            "Epoch 39/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.5974 - accuracy: 0.1429\n",
            "Epoch 40/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.5820 - accuracy: 0.1190\n",
            "Epoch 41/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.5669 - accuracy: 0.1429\n",
            "Epoch 42/100\n",
            "1/1 [==============================] - 0s 27ms/step - loss: 2.5519 - accuracy: 0.1429\n",
            "Epoch 43/100\n",
            "1/1 [==============================] - 0s 26ms/step - loss: 2.5369 - accuracy: 0.1429\n",
            "Epoch 44/100\n",
            "1/1 [==============================] - 0s 24ms/step - loss: 2.5218 - accuracy: 0.1429\n",
            "Epoch 45/100\n",
            "1/1 [==============================] - 0s 26ms/step - loss: 2.5066 - accuracy: 0.1429\n",
            "Epoch 46/100\n",
            "1/1 [==============================] - 0s 25ms/step - loss: 2.4913 - accuracy: 0.1667\n",
            "Epoch 47/100\n",
            "1/1 [==============================] - 0s 23ms/step - loss: 2.4758 - accuracy: 0.1667\n",
            "Epoch 48/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.4601 - accuracy: 0.1667\n",
            "Epoch 49/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.4440 - accuracy: 0.1905\n",
            "Epoch 50/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.4275 - accuracy: 0.2143\n",
            "Epoch 51/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.4105 - accuracy: 0.1905\n",
            "Epoch 52/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.3931 - accuracy: 0.1905\n",
            "Epoch 53/100\n",
            "1/1 [==============================] - 0s 22ms/step - loss: 2.3753 - accuracy: 0.1905\n",
            "Epoch 54/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.3571 - accuracy: 0.1667\n",
            "Epoch 55/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.3386 - accuracy: 0.1667\n",
            "Epoch 56/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.3198 - accuracy: 0.1667\n",
            "Epoch 57/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.3009 - accuracy: 0.1667\n",
            "Epoch 58/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.2819 - accuracy: 0.1667\n",
            "Epoch 59/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.2628 - accuracy: 0.1905\n",
            "Epoch 60/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.2437 - accuracy: 0.1905\n",
            "Epoch 61/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.2246 - accuracy: 0.2143\n",
            "Epoch 62/100\n",
            "1/1 [==============================] - 0s 22ms/step - loss: 2.2056 - accuracy: 0.2143\n",
            "Epoch 63/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.1867 - accuracy: 0.2143\n",
            "Epoch 64/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.1680 - accuracy: 0.2143\n",
            "Epoch 65/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 2.1495 - accuracy: 0.2143\n",
            "Epoch 66/100\n",
            "1/1 [==============================] - 0s 23ms/step - loss: 2.1312 - accuracy: 0.2143\n",
            "Epoch 67/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 2.1133 - accuracy: 0.2143\n",
            "Epoch 68/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 2.0957 - accuracy: 0.2381\n",
            "Epoch 69/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 2.0784 - accuracy: 0.2381\n",
            "Epoch 70/100\n",
            "1/1 [==============================] - 0s 26ms/step - loss: 2.0615 - accuracy: 0.2381\n",
            "Epoch 71/100\n",
            "1/1 [==============================] - 0s 22ms/step - loss: 2.0451 - accuracy: 0.2381\n",
            "Epoch 72/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.0292 - accuracy: 0.2381\n",
            "Epoch 73/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.0138 - accuracy: 0.2381\n",
            "Epoch 74/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.9987 - accuracy: 0.2381\n",
            "Epoch 75/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.9838 - accuracy: 0.2143\n",
            "Epoch 76/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 1.9687 - accuracy: 0.2619\n",
            "Epoch 77/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.9534 - accuracy: 0.2619\n",
            "Epoch 78/100\n",
            "1/1 [==============================] - 0s 22ms/step - loss: 1.9377 - accuracy: 0.2619\n",
            "Epoch 79/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.9216 - accuracy: 0.2857\n",
            "Epoch 80/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.9049 - accuracy: 0.3095\n",
            "Epoch 81/100\n",
            "1/1 [==============================] - 0s 22ms/step - loss: 1.8878 - accuracy: 0.2857\n",
            "Epoch 82/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.8704 - accuracy: 0.2857\n",
            "Epoch 83/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 1.8528 - accuracy: 0.3333\n",
            "Epoch 84/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.8351 - accuracy: 0.3571\n",
            "Epoch 85/100\n",
            "1/1 [==============================] - 0s 22ms/step - loss: 1.8174 - accuracy: 0.3571\n",
            "Epoch 86/100\n",
            "1/1 [==============================] - 0s 24ms/step - loss: 1.7999 - accuracy: 0.3333\n",
            "Epoch 87/100\n",
            "1/1 [==============================] - 0s 26ms/step - loss: 1.7829 - accuracy: 0.3333\n",
            "Epoch 88/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 1.7664 - accuracy: 0.3333\n",
            "Epoch 89/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 1.7505 - accuracy: 0.3333\n",
            "Epoch 90/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 1.7352 - accuracy: 0.3571\n",
            "Epoch 91/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.7205 - accuracy: 0.3810\n",
            "Epoch 92/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 1.7063 - accuracy: 0.3810\n",
            "Epoch 93/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.6926 - accuracy: 0.4048\n",
            "Epoch 94/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 1.6791 - accuracy: 0.4048\n",
            "Epoch 95/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 1.6656 - accuracy: 0.4048\n",
            "Epoch 96/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 1.6521 - accuracy: 0.4286\n",
            "Epoch 97/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 1.6386 - accuracy: 0.4524\n",
            "Epoch 98/100\n",
            "1/1 [==============================] - 0s 21ms/step - loss: 1.6251 - accuracy: 0.4762\n",
            "Epoch 99/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.6117 - accuracy: 0.4762\n",
            "Epoch 100/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 1.5984 - accuracy: 0.4762\n",
            "1/1 [==============================] - 0s 422ms/step\n",
            "\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, LSTM, Embedding\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "\n",
        "# Sample dataset\n",
        "paragraphs = [\n",
        "  \"Je suis allé à la plage avec mes amis et j'ai nagé dans la mer.\",\n",
        "  \"Le chat est monté sur le toit et a attrapé une souris.\",\n",
        "  \"La cuisine française est renommée pour sa gastronomie raffinée.\"\n",
        "]\n",
        "\n",
        "# Function to convert a paragraph to a list of sentences\n",
        "def paragraph_to_sentences(paragraph):\n",
        "    return paragraph.split('. ')\n",
        "\n",
        "# Convert the paragraphs to a list of sentences\n",
        "sentences = []\n",
        "for paragraph in paragraphs:\n",
        "\n",
        "    sentences.extend(paragraph_to_sentences(paragraph))\n",
        "\n",
        "# Tokenize the sentences\n",
        "word2idx = {\"<PAD>\": 0, \"<UNK>\": 1}\n",
        "idx2word = {0: \"<PAD>\", 1: \"<UNK>\"}\n",
        "for sentence in sentences:\n",
        "    for word in sentence.split():\n",
        "        if word not in word2idx:\n",
        "            word2idx[word] = len(word2idx)\n",
        "            idx2word[len(idx2word)] = word\n",
        "\n",
        "# Convert the sentences to sequences of word indices\n",
        "sequences = []\n",
        "for sentence in sentences:\n",
        "    sequence = []\n",
        "    for word in sentence.split():\n",
        "        if word in word2idx:\n",
        "            sequence.append(word2idx[word])\n",
        "        else:\n",
        "            sequence.append(word2idx[\"<UNK>\"])\n",
        "    sequences.append(sequence)\n",
        "\n",
        "# Pad the sequences to a fixed length\n",
        "max_len = max([len(sequence) for sequence in sequences])\n",
        "padded_sequences = pad_sequences(sequences, maxlen=max_len, padding=\"post\", value=word2idx[\"<PAD>\"])\n",
        "\n",
        "# Generate input and output data\n",
        "X = padded_sequences[:, :-1]\n",
        "y = padded_sequences[:, 1:]\n",
        "\n",
        "y_onehot = np.zeros((len(sequences), max_len, len(word2idx)))\n",
        "for i, sequence in enumerate(sequences):\n",
        "    for j, word_idx in enumerate(sequence):\n",
        "        y_onehot[i, j, word_idx] = 1\n",
        "y = y_onehot[:, :-1, :]\n",
        "\n",
        "# Build the LSTM model\n",
        "model = Sequential()\n",
        "model.add(Embedding(input_dim=len(word2idx), output_dim=50, input_length=max_len-1))\n",
        "model.add(LSTM(50, return_sequences=True))\n",
        "model.add(Dense(len(word2idx), activation=\"softmax\"))\n",
        "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
        "# Train the LSTM model\n",
        "model.fit(X, y, epochs=100)\n",
        "\n",
        "# Function to generate a question from a sentence\n",
        "def generate_question(sentence):\n",
        "    # Tokenize the sentence\n",
        "    words = sentence.lower().split()\n",
        "    # Convert the words to word indices\n",
        "    sequence = []\n",
        "    for word in words:\n",
        "        if word in word2idx:\n",
        "            sequence.append(word2idx[word])\n",
        "    # Pad the sequence\n",
        "    sequence = pad_sequences([sequence], maxlen=max_len-1)\n",
        "    # Make the prediction\n",
        "    prediction = model.predict(sequence)\n",
        "    # Convert the prediction to a word\n",
        "    predicted_word_idx = np.argmax(prediction)\n",
        "    predicted_word = idx2word.get(predicted_word_idx, \"\")\n",
        "    # Generate the question\n",
        "    if predicted_word:\n",
        "        question = f\"What is {predicted_word} in the sentence \\\"{sentence}\\\"?\"\n",
        "    else:\n",
        "        question = \"\"\n",
        "    return question\n",
        "\n",
        "# Test the function\n",
        "sentence = \"le ciel est bleu\"\n",
        "\n",
        "question = generate_question(sentence)\n",
        "print(question)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G19BPj4zDcWI"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, LSTM, Embedding\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "import re\n",
        "\n",
        "# Sample dataset in Hindi\n",
        "paragraphs = [\n",
        "  \"이 봉인된 작은 감방의 벽에는 무력한 자의 무력한 동반자처럼 가난이 달라붙어 있습니다.\\\n",
        "지붕이 무너질 정도로 무너져 내렸습니다. 옛날 옛적에 등불이 걸려 있었는데 그 해골이 오늘날에도 여전히 매달려 있습니다.\"\n",
        "]\n",
        "\n",
        "# Function to clean a Hindi sentence\n",
        "def clean_hindi_sentence(sentence):\n",
        "    sentence = re.sub(r'[^\\w\\s]', '', sentence) # Remove punctuations\n",
        "    sentence = re.sub(r'[a-zA-Z]', '', sentence) # Remove English characters\n",
        "    sentence = re.sub(r'\\d+', '', sentence) # Remove digits\n",
        "    sentence = sentence.strip() # Remove leading/trailing whitespaces\n",
        "    return sentence\n",
        "\n",
        "# Function to convert a paragraph to a list of sentences\n",
        "def paragraph_to_sentences(paragraph):\n",
        "    return paragraph.split('। ')\n",
        "\n",
        "# Convert the paragraphs to a list of sentences\n",
        "sentences = []\n",
        "for paragraph in paragraphs:\n",
        "    sentences.extend(paragraph_to_sentences(paragraph))\n",
        "\n",
        "# Clean the sentences\n",
        "cleaned_sentences = [clean_hindi_sentence(sentence) for sentence in sentences]\n",
        "\n",
        "# Tokenize the sentences\n",
        "word2idx = {\"<PAD>\": 0, \"<UNK>\": 1}\n",
        "idx2word = {0: \"<PAD>\", 1: \"<UNK>\"}\n",
        "for sentence in cleaned_sentences:\n",
        "    for word in sentence.split():\n",
        "        if word not in word2idx:\n",
        "            word2idx[word] = len(word2idx)\n",
        "            idx2word[len(idx2word)] = word\n",
        "\n",
        "# Convert the sentences to sequences of word indices\n",
        "sequences = []\n",
        "for sentence in cleaned_sentences:\n",
        "    sequence = []\n",
        "    for word in sentence.split():\n",
        "        if word in word2idx:\n",
        "            sequence.append(word2idx[word])\n",
        "        else:\n",
        "            sequence.append(word2idx[\"<UNK>\"])\n",
        "    sequences.append(sequence)\n",
        "\n",
        "# Pad the sequences to a fixed length\n",
        "max_len = max([len(sequence) for sequence in sequences])\n",
        "padded_sequences = pad_sequences(sequences, maxlen=max_len, padding=\"post\", value=word2idx[\"<PAD>\"])\n",
        "\n",
        "# Generate input and output data\n",
        "X = padded_sequences[:, :-1]\n",
        "y = padded_sequences[:, 1:]\n",
        "\n",
        "y = to_categorical(y, num_classes=len(word2idx))\n",
        "model = Sequential()\n",
        "model.add(Embedding(input_dim=len(word2idx), output_dim=100, input_length=max_len-1))\n",
        "model.add(LSTM(256, return_sequences=True))\n",
        "model.add(Dense(len(word2idx), activation=\"softmax\"))\n",
        "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
        "model.fit(X, y, batch_size=32, epochs=50)\n",
        "seed_text = \"뼈대만 매달려 있는 램프.\"\n",
        "next_words = 10\n",
        "for _ in range(next_words):\n",
        "# Convert the seed text to a sequence of word indices\n",
        "  seed_sequence = []\n",
        "  for word in seed_text.split():\n",
        "    if word in word2idx:\n",
        "      seed_sequence.append(word2idx[word])\n",
        "    else:\n",
        "      seed_sequence.append(word2idx[\"<UNK>\"])\n",
        "# Pad the sequence to a fixed length\n",
        "    seed_padded = pad_sequences([seed_sequence], maxlen=max_len-1, padding=\"post\", value=word2idx[\"<PAD>\"])\n",
        "\n",
        "# Predict the next word\n",
        "predicted = model.predict(seed_padded)[0]\n",
        "predicted_idx = np.argmax(predicted)\n",
        "\n",
        "# Convert the predicted index to a word\n",
        "# Convert the predicted index to a word\n",
        "if predicted_idx in idx2word:\n",
        "    predicted_word = idx2word[predicted_idx]\n",
        "else:\n",
        "    predicted_word = \"<UNK>\"\n",
        "\n",
        "# Update the seed text\n",
        "seed_text += \" \" + predicted_word\n",
        "\n",
        "\n",
        "# Update the seed text\n",
        "seed_text += \" \" + predicted_word\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RCaybtdXMkAK",
        "outputId": "a881ff61-1935-42ec-b00c-325027251993"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "1/1 [==============================] - 2s 2s/step - loss: 2.5090 - accuracy: 0.0000e+00\n",
            "Epoch 2/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.5074 - accuracy: 0.0000e+00\n",
            "Epoch 3/100\n",
            "1/1 [==============================] - 0s 25ms/step - loss: 2.5057 - accuracy: 0.0444\n",
            "Epoch 4/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 2.5041 - accuracy: 0.1111\n",
            "Epoch 5/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.5024 - accuracy: 0.1333\n",
            "Epoch 6/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.5007 - accuracy: 0.1333\n",
            "Epoch 7/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.4990 - accuracy: 0.1111\n",
            "Epoch 8/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.4973 - accuracy: 0.1111\n",
            "Epoch 9/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.4955 - accuracy: 0.1333\n",
            "Epoch 10/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.4936 - accuracy: 0.1111\n",
            "Epoch 11/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.4917 - accuracy: 0.1111\n",
            "Epoch 12/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.4896 - accuracy: 0.1111\n",
            "Epoch 13/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.4875 - accuracy: 0.1111\n",
            "Epoch 14/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.4853 - accuracy: 0.1111\n",
            "Epoch 15/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.4829 - accuracy: 0.1111\n",
            "Epoch 16/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.4804 - accuracy: 0.1333\n",
            "Epoch 17/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.4777 - accuracy: 0.1333\n",
            "Epoch 18/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 2.4749 - accuracy: 0.1333\n",
            "Epoch 19/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.4718 - accuracy: 0.1333\n",
            "Epoch 20/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.4685 - accuracy: 0.1333\n",
            "Epoch 21/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.4649 - accuracy: 0.1333\n",
            "Epoch 22/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.4610 - accuracy: 0.1333\n",
            "Epoch 23/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.4568 - accuracy: 0.1111\n",
            "Epoch 24/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.4522 - accuracy: 0.1111\n",
            "Epoch 25/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 2.4471 - accuracy: 0.1111\n",
            "Epoch 26/100\n",
            "1/1 [==============================] - 0s 23ms/step - loss: 2.4416 - accuracy: 0.1111\n",
            "Epoch 27/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.4354 - accuracy: 0.1111\n",
            "Epoch 28/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.4286 - accuracy: 0.1111\n",
            "Epoch 29/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.4211 - accuracy: 0.1111\n",
            "Epoch 30/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.4129 - accuracy: 0.1111\n",
            "Epoch 31/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.4040 - accuracy: 0.1111\n",
            "Epoch 32/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.3944 - accuracy: 0.1111\n",
            "Epoch 33/100\n",
            "1/1 [==============================] - 0s 22ms/step - loss: 2.3845 - accuracy: 0.1111\n",
            "Epoch 34/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.3746 - accuracy: 0.1111\n",
            "Epoch 35/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.3649 - accuracy: 0.1111\n",
            "Epoch 36/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.3557 - accuracy: 0.1111\n",
            "Epoch 37/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.3469 - accuracy: 0.1111\n",
            "Epoch 38/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.3383 - accuracy: 0.1111\n",
            "Epoch 39/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 2.3298 - accuracy: 0.1111\n",
            "Epoch 40/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.3210 - accuracy: 0.1111\n",
            "Epoch 41/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.3118 - accuracy: 0.1111\n",
            "Epoch 42/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.3019 - accuracy: 0.1111\n",
            "Epoch 43/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.2910 - accuracy: 0.1111\n",
            "Epoch 44/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.2791 - accuracy: 0.1111\n",
            "Epoch 45/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 2.2661 - accuracy: 0.1111\n",
            "Epoch 46/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 2.2520 - accuracy: 0.1111\n",
            "Epoch 47/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.2370 - accuracy: 0.1111\n",
            "Epoch 48/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.2211 - accuracy: 0.0889\n",
            "Epoch 49/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.2047 - accuracy: 0.1111\n",
            "Epoch 50/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.1883 - accuracy: 0.1111\n",
            "Epoch 51/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 2.1723 - accuracy: 0.1111\n",
            "Epoch 52/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.1575 - accuracy: 0.1111\n",
            "Epoch 53/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.1442 - accuracy: 0.1333\n",
            "Epoch 54/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.1323 - accuracy: 0.1556\n",
            "Epoch 55/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 2.1210 - accuracy: 0.1556\n",
            "Epoch 56/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.1099 - accuracy: 0.1556\n",
            "Epoch 57/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.0987 - accuracy: 0.1556\n",
            "Epoch 58/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.0879 - accuracy: 0.1556\n",
            "Epoch 59/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 2.0777 - accuracy: 0.1333\n",
            "Epoch 60/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.0679 - accuracy: 0.1333\n",
            "Epoch 61/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.0583 - accuracy: 0.1333\n",
            "Epoch 62/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.0485 - accuracy: 0.1333\n",
            "Epoch 63/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.0383 - accuracy: 0.1333\n",
            "Epoch 64/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.0273 - accuracy: 0.1333\n",
            "Epoch 65/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 2.0156 - accuracy: 0.1556\n",
            "Epoch 66/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 2.0042 - accuracy: 0.1778\n",
            "Epoch 67/100\n",
            "1/1 [==============================] - 0s 23ms/step - loss: 1.9940 - accuracy: 0.1778\n",
            "Epoch 68/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 1.9845 - accuracy: 0.1778\n",
            "Epoch 69/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.9744 - accuracy: 0.1556\n",
            "Epoch 70/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.9647 - accuracy: 0.1778\n",
            "Epoch 71/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 1.9566 - accuracy: 0.1778\n",
            "Epoch 72/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.9492 - accuracy: 0.1778\n",
            "Epoch 73/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 1.9413 - accuracy: 0.1778\n",
            "Epoch 74/100\n",
            "1/1 [==============================] - 0s 23ms/step - loss: 1.9335 - accuracy: 0.1778\n",
            "Epoch 75/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.9260 - accuracy: 0.1778\n",
            "Epoch 76/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.9175 - accuracy: 0.1778\n",
            "Epoch 77/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 1.9080 - accuracy: 0.2000\n",
            "Epoch 78/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.8992 - accuracy: 0.2000\n",
            "Epoch 79/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.8906 - accuracy: 0.2000\n",
            "Epoch 80/100\n",
            "1/1 [==============================] - 0s 22ms/step - loss: 1.8822 - accuracy: 0.2000\n",
            "Epoch 81/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.8751 - accuracy: 0.2000\n",
            "Epoch 82/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.8678 - accuracy: 0.2000\n",
            "Epoch 83/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 1.8601 - accuracy: 0.2000\n",
            "Epoch 84/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 1.8526 - accuracy: 0.1778\n",
            "Epoch 85/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 1.8443 - accuracy: 0.1778\n",
            "Epoch 86/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.8358 - accuracy: 0.2000\n",
            "Epoch 87/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 1.8275 - accuracy: 0.2000\n",
            "Epoch 88/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.8190 - accuracy: 0.2000\n",
            "Epoch 89/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.8117 - accuracy: 0.2000\n",
            "Epoch 90/100\n",
            "1/1 [==============================] - 0s 20ms/step - loss: 1.8045 - accuracy: 0.2000\n",
            "Epoch 91/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 1.7978 - accuracy: 0.2000\n",
            "Epoch 92/100\n",
            "1/1 [==============================] - 0s 19ms/step - loss: 1.7904 - accuracy: 0.2222\n",
            "Epoch 93/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 1.7826 - accuracy: 0.2222\n",
            "Epoch 94/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 1.7745 - accuracy: 0.2444\n",
            "Epoch 95/100\n",
            "1/1 [==============================] - 0s 16ms/step - loss: 1.7667 - accuracy: 0.2444\n",
            "Epoch 96/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 1.7589 - accuracy: 0.2444\n",
            "Epoch 97/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 1.7516 - accuracy: 0.2444\n",
            "Epoch 98/100\n",
            "1/1 [==============================] - 0s 17ms/step - loss: 1.7445 - accuracy: 0.2444\n",
            "Epoch 99/100\n",
            "1/1 [==============================] - 0s 15ms/step - loss: 1.7373 - accuracy: 0.2444\n",
            "Epoch 100/100\n",
            "1/1 [==============================] - 0s 18ms/step - loss: 1.7293 - accuracy: 0.2444\n",
            "1/1 [==============================] - 0s 448ms/step\n",
            "\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, LSTM, Embedding\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "\n",
        "# Sample dataset\n",
        "paragraphs = [\n",
        "  \"내 가장 친한 친구는 자연이 매우 사랑스럽고 부모님, 담임 선생님, 이웃 등 모두에게 사랑 받고 있습니다.\",\n",
        "  \"그는 시간을 잘 지키고 제 시간에 학교에 옵니다.\",\n",
        "  \"그는 항상 자신의 집안일을 적시에 정기적으로 완료하고 저를 도와줍니다.\"\n",
        "]\n",
        "\n",
        "# Function to convert a paragraph to a list of sentences\n",
        "def paragraph_to_sentences(paragraph):\n",
        "    return paragraph.split('. ')\n",
        "\n",
        "# Convert the paragraphs to a list of sentences\n",
        "sentences = []\n",
        "for paragraph in paragraphs:\n",
        "\n",
        "    sentences.extend(paragraph_to_sentences(paragraph))\n",
        "\n",
        "# Tokenize the sentences\n",
        "word2idx = {\"<PAD>\": 0, \"<UNK>\": 1}\n",
        "idx2word = {0: \"<PAD>\", 1: \"<UNK>\"}\n",
        "for sentence in sentences:\n",
        "    for word in sentence.split():\n",
        "        if word not in word2idx:\n",
        "            word2idx[word] = len(word2idx)\n",
        "            idx2word[len(idx2word)] = word\n",
        "\n",
        "# Convert the sentences to sequences of word indices\n",
        "sequences = []\n",
        "for sentence in sentences:\n",
        "    sequence = []\n",
        "    for word in sentence.split():\n",
        "        if word in word2idx:\n",
        "            sequence.append(word2idx[word])\n",
        "        else:\n",
        "            sequence.append(word2idx[\"<UNK>\"])\n",
        "    sequences.append(sequence)\n",
        "\n",
        "# Pad the sequences to a fixed length\n",
        "max_len = max([len(sequence) for sequence in sequences])\n",
        "padded_sequences = pad_sequences(sequences, maxlen=max_len, padding=\"post\", value=word2idx[\"<PAD>\"])\n",
        "\n",
        "# Generate input and output data\n",
        "X = padded_sequences[:, :-1]\n",
        "y = padded_sequences[:, 1:]\n",
        "\n",
        "y_onehot = np.zeros((len(sequences), max_len, len(word2idx)))\n",
        "for i, sequence in enumerate(sequences):\n",
        "    for j, word_idx in enumerate(sequence):\n",
        "        y_onehot[i, j, word_idx] = 1\n",
        "y = y_onehot[:, :-1, :]\n",
        "\n",
        "# Build the LSTM model\n",
        "model = Sequential()\n",
        "model.add(Embedding(input_dim=len(word2idx), output_dim=50, input_length=max_len-1))\n",
        "model.add(LSTM(50, return_sequences=True))\n",
        "model.add(Dense(len(word2idx), activation=\"softmax\"))\n",
        "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
        "# Train the LSTM model\n",
        "model.fit(X, y, epochs=100)\n",
        "\n",
        "# Function to generate a question from a sentence\n",
        "def generate_question(sentence):\n",
        "    # Tokenize the sentence\n",
        "    words = sentence.lower().split()\n",
        "    # Convert the words to word indices\n",
        "    sequence = []\n",
        "    for word in words:\n",
        "        if word in word2idx:\n",
        "            sequence.append(word2idx[word])\n",
        "    # Pad the sequence\n",
        "    sequence = pad_sequences([sequence], maxlen=max_len-1)\n",
        "    # Make the prediction\n",
        "    prediction = model.predict(sequence)\n",
        "    # Convert the prediction to a word\n",
        "    predicted_word_idx = np.argmax(prediction)\n",
        "    predicted_word = idx2word.get(predicted_word_idx, \"\")\n",
        "    # Generate the question\n",
        "    if predicted_word:\n",
        "        question = f\"What is {predicted_word} in the sentence \\\"{sentence}\\\"?\"\n",
        "    else:\n",
        "        question = \"\"\n",
        "    return question\n",
        "\n",
        "# Test the function\n",
        "sentence = \"그는 항상 자신의 집안일을 적시에 정기적으로 완료하고 저를 도와줍니다.\"\n",
        "\n",
        "question = generate_question(sentence)\n",
        "print(question)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "axwMJGNAPtHc"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, LSTM, Embedding\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "\n",
        "# Sample dataset\n",
        "paragraphs = [\n",
        "  \"The sky is blue. The grass is green.\",\n",
        "  \"The quick brown fox jumps over the lazy dog.\",\n",
        "  \"Roses are red. Violets are blue.\"\n",
        "]\n",
        "\n",
        "# Function to convert a paragraph to a list of sentences\n",
        "def paragraph_to_sentences(paragraph):\n",
        "    return paragraph.split('. ')\n",
        "\n",
        "# Convert the paragraphs to a list of sentences\n",
        "sentences = []\n",
        "for paragraph in paragraphs:\n",
        "\n",
        "    sentences.extend(paragraph_to_sentences(paragraph))\n",
        "\n",
        "# Tokenize the sentences\n",
        "word2idx = {\"<PAD>\": 0, \"<UNK>\": 1}\n",
        "idx2word = {0: \"<PAD>\", 1: \"<UNK>\"}\n",
        "for sentence in sentences:\n",
        "    for word in sentence.split():\n",
        "        if word not in word2idx:\n",
        "            word2idx[word] = len(word2idx)\n",
        "            idx2word[len(idx2word)] = word\n",
        "\n",
        "# Convert the sentences to sequences of word indices\n",
        "sequences = []\n",
        "for sentence in sentences:\n",
        "    sequence = []\n",
        "    for word in sentence.split():\n",
        "        if word in word2idx:\n",
        "            sequence.append(word2idx[word])\n",
        "        else:\n",
        "            sequence.append(word2idx[\"<UNK>\"])\n",
        "    sequences.append(sequence)\n",
        "\n",
        "# Pad the sequences to a fixed length\n",
        "max_len = max([len(sequence) for sequence in sequences])\n",
        "padded_sequences = pad_sequences(sequences, maxlen=max_len, padding=\"post\", value=word2idx[\"<PAD>\"])\n",
        "\n",
        "# Generate input and output data\n",
        "X = padded_sequences[:, :-1]\n",
        "y = padded_sequences[:, 1:]\n",
        "\n",
        "y_onehot = np.zeros((len(sequences), max_len, len(word2idx)))\n",
        "for i, sequence in enumerate(sequences):\n",
        "    for j, word_idx in enumerate(sequence):\n",
        "        y_onehot[i, j, word_idx] = 1\n",
        "y = y_onehot[:, :-1, :]\n",
        "\n",
        "# Build the LSTM model\n",
        "model = Sequential()\n",
        "model.add(Embedding(input_dim=len(word2idx), output_dim=50, input_length=max_len-1))\n",
        "model.add(LSTM(50, return_sequences=True))\n",
        "model.add(Dense(len(word2idx), activation=\"softmax\"))\n",
        "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
        "# Train the LSTM model\n",
        "model.fit(X, y, epochs=100)\n",
        "\n",
        "# Function to generate a question from a sentence\n",
        "def generate_question(sentence):\n",
        "    # Tokenize the sentence\n",
        "    words = sentence.lower().split()\n",
        "    # Convert the words to word indices\n",
        "    sequence = []\n",
        "    for word in words:\n",
        "        if word in word2idx:\n",
        "            sequence.append(word2idx[word])\n",
        "    # Pad the sequence\n",
        "    sequence = pad_sequences([sequence], maxlen=max_len-1)\n",
        "    # Make the prediction\n",
        "    prediction = model.predict(sequence)\n",
        "    # Convert the prediction to a word\n",
        "    predicted_word_idx = np.argmax(prediction)\n",
        "    predicted_word = idx2word.get(predicted_word_idx, \"\")\n",
        "    # Generate the question\n",
        "    if predicted_word:\n",
        "        question = f\"What is {predicted_word} in the sentence \\\"{sentence}\\\"?\"\n",
        "    else:\n",
        "        question = \"\"\n",
        "    return question\n",
        "\n",
        "# Test the function\n",
        "sentence = \"The sky is blue.\"\n",
        "question = generate_question(sentence)\n",
        "print(question)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yHglcEV6PZk8",
        "outputId": "be85f1cd-68ca-4852-d574-b86f3d4bae80"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers\n",
            "pip install xformers.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: googletrans==3.1.0a0 in /usr/local/lib/python3.10/dist-packages (3.1.0a0)\n",
            "Requirement already satisfied: httpx==0.13.3 in /usr/local/lib/python3.10/dist-packages (from googletrans==3.1.0a0) (0.13.3)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx==0.13.3->googletrans==3.1.0a0) (2023.5.7)\n",
            "Requirement already satisfied: hstspreload in /usr/local/lib/python3.10/dist-packages (from httpx==0.13.3->googletrans==3.1.0a0) (2023.1.1)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx==0.13.3->googletrans==3.1.0a0) (1.3.0)\n",
            "Requirement already satisfied: chardet==3.* in /usr/local/lib/python3.10/dist-packages (from httpx==0.13.3->googletrans==3.1.0a0) (3.0.4)\n",
            "Requirement already satisfied: idna==2.* in /usr/local/lib/python3.10/dist-packages (from httpx==0.13.3->googletrans==3.1.0a0) (2.10)\n",
            "Requirement already satisfied: rfc3986<2,>=1.3 in /usr/local/lib/python3.10/dist-packages (from httpx==0.13.3->googletrans==3.1.0a0) (1.5.0)\n",
            "Requirement already satisfied: httpcore==0.9.* in /usr/local/lib/python3.10/dist-packages (from httpx==0.13.3->googletrans==3.1.0a0) (0.9.1)\n",
            "Requirement already satisfied: h11<0.10,>=0.8 in /usr/local/lib/python3.10/dist-packages (from httpcore==0.9.*->httpx==0.13.3->googletrans==3.1.0a0) (0.9.0)\n",
            "Requirement already satisfied: h2==3.* in /usr/local/lib/python3.10/dist-packages (from httpcore==0.9.*->httpx==0.13.3->googletrans==3.1.0a0) (3.2.0)\n",
            "Requirement already satisfied: hyperframe<6,>=5.2.0 in /usr/local/lib/python3.10/dist-packages (from h2==3.*->httpcore==0.9.*->httpx==0.13.3->googletrans==3.1.0a0) (5.2.0)\n",
            "Requirement already satisfied: hpack<4,>=3.0 in /usr/local/lib/python3.10/dist-packages (from h2==3.*->httpcore==0.9.*->httpx==0.13.3->googletrans==3.1.0a0) (3.0.0)\n",
            "[]\n",
            "0\n"
          ]
        }
      ],
      "source": [
        "import itertools\n",
        "import logging\n",
        "from typing import Optional, Dict, Union\n",
        "from nltk import sent_tokenize\n",
        "from langdetect import detect\n",
        "import torch\n",
        "from transformers import(\n",
        "    AutoModelForSeq2SeqLM,\n",
        "    AutoTokenizer,\n",
        "    PreTrainedModel,\n",
        "    PreTrainedTokenizer,\n",
        ")\n",
        "logger = logging.getLogger(__name__)\n",
        "import random\n",
        "import nltk\n",
        "from nltk.corpus import wordnet\n",
        "from transformers import pipeline\n",
        "nltk.download('wordnet')\n",
        "generator = pipeline('text-generation', model='gpt2')\n",
        "!pip install googletrans==3.1.0a0\n",
        "from googletrans import Translator\n",
        "ans=[]\n",
        "class QGPipeline:\n",
        "    \"\"\"Poor man's QG pipeline\"\"\"\n",
        "    def __init__(\n",
        "        self,\n",
        "        model: PreTrainedModel,\n",
        "        tokenizer: PreTrainedTokenizer,\n",
        "        ans_model: PreTrainedModel,\n",
        "        ans_tokenizer: PreTrainedTokenizer,\n",
        "        qg_format: str,\n",
        "        use_cuda: bool\n",
        "    ):\n",
        "        self.model = model\n",
        "        self.tokenizer = tokenizer\n",
        "\n",
        "        self.ans_model = ans_model\n",
        "        self.ans_tokenizer = ans_tokenizer\n",
        "\n",
        "        self.qg_format = qg_format\n",
        "\n",
        "        self.device = \"cuda\" if torch.cuda.is_available() and use_cuda else \"cpu\"\n",
        "        self.model.to(self.device)\n",
        "\n",
        "        if self.ans_model is not self.model:\n",
        "            self.ans_model.to(self.device)\n",
        "\n",
        "        assert self.model.__class__.__name__ in [\"T5ForConditionalGeneration\", \"BartForConditionalGeneration\"]\n",
        "\n",
        "        if \"T5ForConditionalGeneration\" in self.model.__class__.__name__:\n",
        "            self.model_type = \"t5\"\n",
        "        else:\n",
        "            self.model_type = \"bart\"\n",
        "\n",
        "    #def generate_distractors(target_word, num_distractors):\n",
        "        # Find synonyms, antonyms, hyponyms, and hypernyms of the target word\n",
        "\n",
        "\n",
        "    def __call__(self, inputs: str):\n",
        "        f=detect(inputs)\n",
        "        t = Translator()\n",
        "        x = t.translate(inputs)\n",
        "        inputs=x.text\n",
        "        inputs = \" \".join(inputs.split())\n",
        "        sents, answers = self._extract_answers(inputs)\n",
        "        flat_answers = list(itertools.chain(*answers))\n",
        "\n",
        "        if len(flat_answers) == 0:\n",
        "          return []\n",
        "\n",
        "        if self.qg_format == \"prepend\":\n",
        "            qg_examples = self._prepare_inputs_for_qg_from_answers_prepend(inputs, answers)\n",
        "        else:\n",
        "            qg_examples = self._prepare_inputs_for_qg_from_answers_hl(sents, answers)\n",
        "\n",
        "        qg_inputs = [example['source_text'] for example in qg_examples]\n",
        "        questions = self._generate_questions(qg_inputs)\n",
        "        for example, que in zip(qg_examples, questions):\n",
        "\n",
        "            #distractors = generate_distractors(example['answer'], 3)\n",
        "            if len(example['answer'])==0:\n",
        "              continue\n",
        "            words=example['answer'].split()\n",
        "            last_word = words[-1]\n",
        "            target_word=last_word\n",
        "            num_distractors=3\n",
        "            synonyms = set()\n",
        "            antonyms = set()\n",
        "            hyponyms = set()\n",
        "            hypernyms = set()\n",
        "\n",
        "            for syn in wordnet.synsets(target_word):\n",
        "                for lemma in syn.lemmas():\n",
        "                    synonyms.add(lemma.name())\n",
        "                    if lemma.antonyms():\n",
        "                        antonyms.add(lemma.antonyms()[0].name())\n",
        "                for hypo in syn.hyponyms():\n",
        "                    for lemma in hypo.lemmas():\n",
        "                        hyponyms.add(lemma.name())\n",
        "                for hyper in syn.hypernyms():\n",
        "                    for lemma in hyper.lemmas():\n",
        "                        hypernyms.add(lemma.name())\n",
        "\n",
        "            # Generate candidate distractors using the language model\n",
        "            candidates = list(synonyms.union(antonyms).union(hyponyms).union(hypernyms))\n",
        "            distractors = []\n",
        "            for candidate in candidates:\n",
        "                if candidate != target_word:\n",
        "                    try:\n",
        "                        generated_text = generator(f\"Which is more related to {target_word}? {target_word} or {candidate}\", max_length=20, num_return_sequences=1, do_sample=True)[0]['generated_text'].strip()\n",
        "                        distractors.append((candidate, generated_text))\n",
        "                    except:\n",
        "                        pass\n",
        "\n",
        "            # Rank distractors by relevance and choose top N\n",
        "            distractors.sort(key=lambda x: x[1])\n",
        "            xx = t.translate(example['answer'],dest=f)\n",
        "            example['answer']=xx.text\n",
        "            xy=t.translate(que,dest=f)\n",
        "            que=xy.text\n",
        "            distractors= [d[0] for d in distractors[:num_distractors]]\n",
        "            if len(distractors)==0 and f=='hi':\n",
        "              distractors.append('जिजीविषा')\n",
        "              distractors.append('प्रेमशक्त')\n",
        "              distractors.append('तमक')\n",
        "            elif len(distractors)==0:\n",
        "              distractors.append('Morrow')\n",
        "              distractors.append('Kerfuffle')\n",
        "              distractors.append('Crapulous')\n",
        "            for m in range(len(distractors)):\n",
        "              mm=t.translate(distractors[m],dest=f)\n",
        "              distractors[m]=mm.text\n",
        "            index = random.randint(0, len(distractors))\n",
        "            distractors.insert(index,example['answer'])\n",
        "            output=[{'question': que, 'distractors':distractors,'answer': example['answer']}]\n",
        "            ans.append(output)\n",
        "            #print(output)\n",
        "        for i in range(len(ans)):\n",
        "            print(ans[i])\n",
        "        ans1=ans.copy()\n",
        "        ans.clear()\n",
        "        #print(ans)\n",
        "        return ans1\n",
        "    def _generate_questions(self, inputs):\n",
        "        inputs = self._tokenize(inputs, padding=True, truncation=True)\n",
        "\n",
        "        outs = self.model.generate(\n",
        "            input_ids=inputs['input_ids'].to(self.device),\n",
        "            attention_mask=inputs['attention_mask'].to(self.device),\n",
        "            max_length=32,\n",
        "            num_beams=4,\n",
        "        )\n",
        "\n",
        "        questions = [self.tokenizer.decode(ids, skip_special_tokens=True) for ids in outs]\n",
        "        return questions\n",
        "\n",
        "    def _extract_answers(self, context):\n",
        "        sents, inputs = self._prepare_inputs_for_ans_extraction(context)\n",
        "        inputs = self._tokenize(inputs, padding=True, truncation=True)\n",
        "\n",
        "        outs = self.ans_model.generate(\n",
        "            input_ids=inputs['input_ids'].to(self.device),\n",
        "            attention_mask=inputs['attention_mask'].to(self.device),\n",
        "            max_length=32,\n",
        "        )\n",
        "\n",
        "        dec = [self.ans_tokenizer.decode(ids, skip_special_tokens=False) for ids in outs]\n",
        "        answers = [item.split('<sep>') for item in dec]\n",
        "        answers = [i[:-1] for i in answers]\n",
        "\n",
        "        return sents, answers\n",
        "\n",
        "    def _tokenize(self,\n",
        "        inputs,\n",
        "        padding=True,\n",
        "        truncation=True,\n",
        "        add_special_tokens=True,\n",
        "        max_length=512\n",
        "    ):\n",
        "        inputs = self.tokenizer.batch_encode_plus(\n",
        "            inputs,\n",
        "            max_length=max_length,\n",
        "            add_special_tokens=add_special_tokens,\n",
        "            truncation=truncation,\n",
        "            padding=\"max_length\" if padding else False,\n",
        "            pad_to_max_length=padding,\n",
        "            return_tensors=\"pt\"\n",
        "        )\n",
        "        return inputs\n",
        "\n",
        "    def _prepare_inputs_for_ans_extraction(self, text):\n",
        "        sents = sent_tokenize(text)\n",
        "\n",
        "        inputs = []\n",
        "        for i in range(len(sents)):\n",
        "            source_text = \"extract answers:\"\n",
        "            for j, sent in enumerate(sents):\n",
        "                if i == j:\n",
        "                    sent = \"<hl> %s <hl>\" % sent\n",
        "                source_text = \"%s %s\" % (source_text, sent)\n",
        "                source_text = source_text.strip()\n",
        "\n",
        "            if self.model_type == \"t5\":\n",
        "                source_text = source_text + \" </s>\"\n",
        "            inputs.append(source_text)\n",
        "\n",
        "        return sents, inputs\n",
        "\n",
        "    def _prepare_inputs_for_qg_from_answers_hl(self, sents, answers):\n",
        "        inputs = []\n",
        "        #print(answers)\n",
        "        #print(sents)\n",
        "        for i, answer in enumerate(answers):\n",
        "            if len(answer) == 0:\n",
        "              continue\n",
        "            for answer_text in answer:\n",
        "                sent = sents[i]\n",
        "                sent=sent.lower()\n",
        "                sents_copy = sents[:]\n",
        "                answer_text=answer_text[5:]\n",
        "                answer_text=answer_text.lower()\n",
        "                #print(answer_text)\n",
        "                answer_text = answer_text.strip()\n",
        "\n",
        "                ans_start_idx = sent.index(answer_text)\n",
        "\n",
        "                sent = f\"{sent[:ans_start_idx]} <hl> {answer_text} <hl> {sent[ans_start_idx + len(answer_text): ]}\"\n",
        "                sents_copy[i] = sent\n",
        "\n",
        "                source_text = \" \".join(sents_copy)\n",
        "                source_text = f\"generate question: {source_text}\"\n",
        "                if self.model_type == \"t5\":\n",
        "                    source_text = source_text + \" </s>\"\n",
        "\n",
        "                inputs.append({\"answer\": answer_text, \"source_text\": source_text})\n",
        "\n",
        "        return inputs\n",
        "\n",
        "    def _prepare_inputs_for_qg_from_answers_prepend(self, context, answers):\n",
        "        flat_answers = list(itertools.chain(*answers))\n",
        "        examples = []\n",
        "        for answer in flat_answers:\n",
        "            source_text = f\"answer: {answer} context: {context}\"\n",
        "            if self.model_type == \"t5\":\n",
        "                source_text = source_text + \" </s>\"\n",
        "\n",
        "            examples.append({\"answer\": answer, \"source_text\": source_text})\n",
        "        return examples\n",
        "\n",
        "\n",
        "class MultiTaskQAQGPipeline(QGPipeline):\n",
        "    def __init__(self, **kwargs):\n",
        "        super().__init__(**kwargs)\n",
        "\n",
        "    def __call__(self, inputs: Union[Dict, str]):\n",
        "        if type(inputs) is str:\n",
        "            # do qg\n",
        "            return super().__call__(inputs)\n",
        "        else:\n",
        "            # do qa\n",
        "            return self._extract_answer(inputs[\"question\"], inputs[\"context\"])\n",
        "\n",
        "    def _prepare_inputs_for_qa(self, question, context):\n",
        "        source_text = f\"question: {question}  context: {context}\"\n",
        "        if self.model_type == \"t5\":\n",
        "            source_text = source_text + \" </s>\"\n",
        "        return  source_text\n",
        "\n",
        "    def _extract_answer(self, question, context):\n",
        "        source_text = self._prepare_inputs_for_qa(question, context)\n",
        "        inputs = self._tokenize([source_text], padding=False)\n",
        "\n",
        "        outs = self.model.generate(\n",
        "            input_ids=inputs['input_ids'].to(self.device),\n",
        "            attention_mask=inputs['attention_mask'].to(self.device),\n",
        "            max_length=16,\n",
        "        )\n",
        "\n",
        "        answer = self.tokenizer.decode(outs[0], skip_special_tokens=True)\n",
        "        return answer\n",
        "\n",
        "\n",
        "class E2EQGPipeline:\n",
        "    def __init__(\n",
        "        self,\n",
        "        model: PreTrainedModel,\n",
        "        tokenizer: PreTrainedTokenizer,\n",
        "        use_cuda: bool\n",
        "    ) :\n",
        "\n",
        "        self.model = model\n",
        "        self.tokenizer = tokenizer\n",
        "\n",
        "        self.device = \"cuda\" if torch.cuda.is_available() and use_cuda else \"cpu\"\n",
        "        self.model.to(self.device)\n",
        "\n",
        "        assert self.model.__class__.__name__ in [\"T5ForConditionalGeneration\", \"BartForConditionalGeneration\"]\n",
        "\n",
        "        if \"T5ForConditionalGeneration\" in self.model.__class__.__name__:\n",
        "            self.model_type = \"t5\"\n",
        "        else:\n",
        "            self.model_type = \"bart\"\n",
        "\n",
        "        self.default_generate_kwargs = {\n",
        "            \"max_length\": 256,\n",
        "            \"num_beams\": 4,\n",
        "            \"length_penalty\": 1.5,\n",
        "            \"no_repeat_ngram_size\": 3,\n",
        "            \"early_stopping\": True,\n",
        "        }\n",
        "\n",
        "    def __call__(self, context: str, **generate_kwargs):\n",
        "        inputs = self._prepare_inputs_for_e2e_qg(context)\n",
        "\n",
        "        # TODO: when overrding default_generate_kwargs all other arguments need to be passsed\n",
        "        # find a better way to do this\n",
        "        if not generate_kwargs:\n",
        "            generate_kwargs = self.default_generate_kwargs\n",
        "\n",
        "        input_length = inputs[\"input_ids\"].shape[-1]\n",
        "\n",
        "        # max_length = generate_kwargs.get(\"max_length\", 256)\n",
        "        # if input_length < max_length:\n",
        "        #     logger.warning(\n",
        "        #         \"Your max_length is set to {}, but you input_length is only {}. You might consider decreasing max_length manually, e.g. summarizer('...', max_length=50)\".format(\n",
        "        #             max_length, input_length\n",
        "        #         )\n",
        "        #     )\n",
        "\n",
        "        outs = self.model.generate(\n",
        "            input_ids=inputs['input_ids'].to(self.device),\n",
        "            attention_mask=inputs['attention_mask'].to(self.device),\n",
        "            **generate_kwargs\n",
        "        )\n",
        "\n",
        "        prediction = self.tokenizer.decode(outs[0], skip_special_tokens=True)\n",
        "        questions = prediction.split(\"<sep>\")\n",
        "        questions = [question.strip() for question in questions[:-1]]\n",
        "        return questions\n",
        "\n",
        "    def _prepare_inputs_for_e2e_qg(self, context):\n",
        "        source_text = f\"generate questions: {context}\"\n",
        "        if self.model_type == \"t5\":\n",
        "            source_text = source_text + \" </s>\"\n",
        "\n",
        "        inputs = self._tokenize([source_text], padding=False)\n",
        "        return inputs\n",
        "\n",
        "    def _tokenize(\n",
        "        self,\n",
        "        inputs,\n",
        "        padding=True,\n",
        "        truncation=True,\n",
        "        add_special_tokens=True,\n",
        "        max_length=512\n",
        "    ):\n",
        "        inputs = self.tokenizer.batch_encode_plus(\n",
        "            inputs,\n",
        "            max_length=max_length,\n",
        "            add_special_tokens=add_special_tokens,\n",
        "            truncation=truncation,\n",
        "            padding=\"max_length\" if padding else False,\n",
        "            pad_to_max_length=padding,\n",
        "            return_tensors=\"pt\"\n",
        "        )\n",
        "        return inputs\n",
        "\n",
        "\n",
        "SUPPORTED_TASKS = {\n",
        "    \"question-generation\": {\n",
        "        \"impl\": QGPipeline,\n",
        "        \"default\": {\n",
        "            \"model\": \"valhalla/t5-small-qg-hl\",\n",
        "            \"ans_model\": \"valhalla/t5-base-qa-qg-hl\",\n",
        "        }\n",
        "    },\n",
        "    \"multitask-qa-qg\": {\n",
        "        \"impl\": MultiTaskQAQGPipeline,\n",
        "        \"default\": {\n",
        "            \"model\": \"valhalla/t5-base-qa-qg-hl\",\n",
        "        }\n",
        "    },\n",
        "    \"e2e-qg\": {\n",
        "        \"impl\": E2EQGPipeline,\n",
        "        \"default\": {\n",
        "            \"model\": \"valhalla/t5-small-e2e-qg\",\n",
        "        }\n",
        "    }\n",
        "}\n",
        "\n",
        "def pipeline(\n",
        "    task: str,\n",
        "    model = None,\n",
        "    tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None,\n",
        "    qg_format: Optional[str] = \"highlight\",\n",
        "    ans_model = None,\n",
        "    ans_tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None,\n",
        "    use_cuda: Optional[bool] = True,\n",
        "    **kwargs,\n",
        "):\n",
        "    # Retrieve the task\n",
        "    if task not in SUPPORTED_TASKS:\n",
        "        raise KeyError(\"Unknown task {}, available tasks are {}\".format(task, list(SUPPORTED_TASKS.keys())))\n",
        "\n",
        "    targeted_task = SUPPORTED_TASKS[task]\n",
        "    task_class = targeted_task[\"impl\"]\n",
        "\n",
        "    # Use default model/config/tokenizer for the task if no model is provided\n",
        "    if model is None:\n",
        "        model = targeted_task[\"default\"][\"model\"]\n",
        "\n",
        "    # Try to infer tokenizer from model or config name (if provided as str)\n",
        "    if tokenizer is None:\n",
        "        if isinstance(model, str):\n",
        "            tokenizer = model\n",
        "        else:\n",
        "            # Impossible to guest what is the right tokenizer here\n",
        "            raise Exception(\n",
        "                \"Impossible to guess which tokenizer to use. \"\n",
        "                \"Please provided a PretrainedTokenizer class or a path/identifier to a pretrained tokenizer.\"\n",
        "            )\n",
        "\n",
        "    # Instantiate tokenizer if needed\n",
        "    if isinstance(tokenizer, (str, tuple)):\n",
        "        if isinstance(tokenizer, tuple):\n",
        "            # For tuple we have (tokenizer name, {kwargs})\n",
        "            tokenizer = AutoTokenizer.from_pretrained(tokenizer[0], **tokenizer[1])\n",
        "        else:\n",
        "            tokenizer = AutoTokenizer.from_pretrained(tokenizer)\n",
        "\n",
        "    # Instantiate model if needed\n",
        "    if isinstance(model, str):\n",
        "        model = AutoModelForSeq2SeqLM.from_pretrained(model)\n",
        "\n",
        "    if task == \"question-generation\":\n",
        "        if ans_model is None:\n",
        "            # load default ans model\n",
        "            ans_model = targeted_task[\"default\"][\"ans_model\"]\n",
        "            ans_tokenizer = AutoTokenizer.from_pretrained(ans_model)\n",
        "            ans_model = AutoModelForSeq2SeqLM.from_pretrained(ans_model)\n",
        "        else:\n",
        "            # Try to infer tokenizer from model or config name (if provided as str)\n",
        "            if ans_tokenizer is None:\n",
        "                if isinstance(ans_model, str):\n",
        "                    ans_tokenizer = ans_model\n",
        "                else:\n",
        "                    # Impossible to guest what is the right tokenizer here\n",
        "                    raise Exception(\n",
        "                        \"Impossible to guess which tokenizer to use. \"\n",
        "                        \"Please provided a PretrainedTokenizer class or a path/identifier to a pretrained tokenizer.\"\n",
        "                    )\n",
        "\n",
        "            # Instantiate tokenizer if needed\n",
        "            if isinstance(ans_tokenizer, (str, tuple)):\n",
        "                if isinstance(ans_tokenizer, tuple):\n",
        "                    # For tuple we have (tokenizer name, {kwargs})\n",
        "                    ans_tokenizer = AutoTokenizer.from_pretrained(ans_tokenizer[0], **ans_tokenizer[1])\n",
        "                else:\n",
        "                    ans_tokenizer = AutoTokenizer.from_pretrained(ans_tokenizer)\n",
        "\n",
        "            if isinstance(ans_model, str):\n",
        "                ans_model = AutoModelForSeq2SeqLM.from_pretrained(ans_model)\n",
        "\n",
        "    if task == \"e2e-qg\":\n",
        "        return task_class(model=model, tokenizer=tokenizer, use_cuda=use_cuda)\n",
        "    elif task == \"question-generation\":\n",
        "        return task_class(model=model, tokenizer=tokenizer, ans_model=ans_model, ans_tokenizer=ans_tokenizer, qg_format=qg_format, use_cuda=use_cuda)\n",
        "    else:\n",
        "        return task_class(model=model, tokenizer=tokenizer, ans_model=model, ans_tokenizer=tokenizer, qg_format=qg_format, use_cuda=use_cuda)\n",
        "print(ans)\n",
        "print(len(ans))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SFSZiIc0StHY"
      },
      "outputs": [],
      "source": [
        "nlp = pipeline(\"question-generation\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vbDVLXDpP0YQ"
      },
      "outputs": [],
      "source": [
        "answers = [['<pad> Python'], ['<pad> Guido van Rossum']]\n",
        "sents = ['Python is an interpreted, high-level, general-purpose programming language.', \"Created by Guido van Rossum and first released in 1991, Python's design philosophy emphasizes code readability with its notable use of significant whitespace.\"]\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "furVTILRP3SN",
        "outputId": "4c3059e5-7281-4a1d-fcf0-64e57331284d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['<pad> Python']\n",
            "<pad> Python\n",
            "Python\n",
            "Python is an interpreted, high-level, general-purpose programming language.\n",
            "['<pad> Guido van Rossum']\n",
            "<pad> Guido van Rossum\n",
            "Guido\n",
            "Created by Guido van Rossum and first released in 1991, Python's design philosophy emphasizes code readability with its notable use of significant whitespace.\n"
          ]
        }
      ],
      "source": [
        "inputs = []\n",
        "# print(answers)\n",
        "# print(sents)\n",
        "for i, answer in enumerate(answers):\n",
        "  print(answer)\n",
        "  if len(answer) == 0: continue\n",
        "  for answer_text in answer:\n",
        "      print(answer_text)\n",
        "      sent = sents[i]\n",
        "      sents_copy = sents[:]\n",
        "\n",
        "      answer_text = answer_text.split(\" \")[1]\n",
        "      print(answer_text)\n",
        "      print(sent)\n",
        "      ans_start_idx = sent.index(answer_text)\n",
        "\n",
        "      sent = f\"{sent[:ans_start_idx]} <hl> {answer_text} <hl> {sent[ans_start_idx + len(answer_text): ]}\"\n",
        "      sents_copy[i] = sent\n",
        "\n",
        "      source_text = \" \".join(sents_copy)\n",
        "      source_text = f\"generate question: {source_text}\"\n",
        "      # if self.model_type == \"t5\":\n",
        "      #     source_text = source_text + \" </s>\"\n",
        "\n",
        "      inputs.append({\"answer\": answer_text, \"source_text\": source_text})"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1DHB0dDqTb-o"
      },
      "source": [
        "If you want to use the t5-base model, then pass the path through model parameter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_050CddNTWeU"
      },
      "outputs": [],
      "source": [
        "nlp = pipeline(\"question-generation\", model=\"valhalla/t5-base-qg-hl\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 256
        },
        "id": "xn0vU0TISMss",
        "outputId": "09459fa0-d3d3-44f4-ccd6-3c3e7b5529ac"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: anvil-uplink in /usr/local/lib/python3.10/dist-packages (0.4.2)\n",
            "Collecting argparse (from anvil-uplink)\n",
            "  Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.10/dist-packages (from anvil-uplink) (0.18.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from anvil-uplink) (1.16.0)\n",
            "Requirement already satisfied: ws4py in /usr/local/lib/python3.10/dist-packages (from anvil-uplink) (0.5.1)\n",
            "Installing collected packages: argparse\n",
            "Successfully installed argparse-1.4.0\n"
          ]
        },
        {
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "argparse"
                ]
              }
            }
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "!pip install anvil-uplink"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eWIo-wTzShqu",
        "outputId": "3b06abeb-8e39-4cd5-eb7b-780cc44b4316"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Connecting to wss://anvil.works/uplink\n",
            "Anvil websocket open\n",
            "Connected to \"Default Environment\" as SERVER\n"
          ]
        }
      ],
      "source": [
        "import anvil.server\n",
        "anvil.server.connect(\"server_T7AFCZLZIBPXCTTIJJ5IQLPB-JARBJ3BJAHU5SSSZ\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AzSg34gP5Kx1"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q7EkjnW65Ku8"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-ilhtoyzsp32"
      },
      "outputs": [],
      "source": [
        "def add_backslash_after_word_count(text, word_count):\n",
        "    words = text.split()\n",
        "    modified_text = ''\n",
        "    for i, word in enumerate(words, 1):\n",
        "        modified_text += word\n",
        "        if i % word_count == 0:\n",
        "            modified_text += '\\\\'\n",
        "        else:\n",
        "            modified_text += ' '\n",
        "    return modified_text\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ArSHFtl7yUsN"
      },
      "outputs": [],
      "source": [
        "mylist = []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NTKjWuZ9h0ux"
      },
      "outputs": [],
      "source": [
        "# def format_data(data):\n",
        "#     result = \"\"\n",
        "#     for element in data:\n",
        "#         if isinstance(element, dict):\n",
        "#             for key, value in element.items():\n",
        "#                 result += f\"{key}: {value}\\n\"\n",
        "#             result += \"\\n\"\n",
        "#     return result\n",
        "\n",
        "def format_data(data):\n",
        "    result = \"\"\n",
        "    for element in data:\n",
        "        if isinstance(element, list):\n",
        "            for dictionary in element:\n",
        "                for key, value in dictionary.items():\n",
        "                    if isinstance(value, list):\n",
        "                        value_str = ', '.join(map(str, value))\n",
        "                        result += f\"{key}: [{value_str}]\\n\"\n",
        "                    else:\n",
        "                        result += f\"{key}: {value}\\n\"\n",
        "                result += \"\\n\"\n",
        "        elif isinstance(element, dict):\n",
        "            for key, value in element.items():\n",
        "                if isinstance(value, list):\n",
        "                    value_str = ', '.join(map(str, value))\n",
        "                    result += f\"{key}: [{value_str}]\\n\"\n",
        "                else:\n",
        "                    result += f\"{key}: {value}\\n\"\n",
        "            result += \"\\n\"\n",
        "    return result"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P_TEnfiESvsd"
      },
      "outputs": [],
      "source": [
        "@anvil.server.callable\n",
        "def gen_ques (text):\n",
        "  print(text)\n",
        "  print('hello')\n",
        "  # modified_text=add_backslash_after_word_count(text,10)\n",
        "  print(text)\n",
        "  return(format_data(nlp(text)))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "vqwTyOiTTh8M",
        "outputId": "d9d3c79f-b920-4eb8-f1e5-960a88e88b74"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Uniform Civil Code is a proposal to create and enforce personal laws of citizens in India that apply equally to all citizens regardless of their religion, gender and sexual orientation. At present, the personal laws of different communities are governed by their religious texts. The implementation of a Uniform Civil Code across the country is one of the controversial promises made by India's ruling Bharatiya Janata Party.It is an important issue regarding secularism in Indian politics and remains disputed by India's political left, Muslim groups and other conservative religious groups and sects in defense of Sharia and religious customs. Personal law is different from public law and covers marriage, divorce. , inheritance, adoption and maintenance.\n",
            "hello\n",
            "Uniform Civil Code is a proposal to create and enforce personal laws of citizens in India that apply equally to all citizens regardless of their religion, gender and sexual orientation. At present, the personal laws of different communities are governed by their religious texts. The implementation of a Uniform Civil Code across the country is one of the controversial promises made by India's ruling Bharatiya Janata Party.It is an important issue regarding secularism in Indian politics and remains disputed by India's political left, Muslim groups and other conservative religious groups and sects in defense of Sharia and religious customs. Personal law is different from public law and covers marriage, divorce. , inheritance, adoption and maintenance.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Input length of input_ids is 21, but `max_length` is set to 20. This can lead to unexpected behavior. You should consider increasing `max_new_tokens`.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[{'question': 'What is a proposal to create and enforce personal laws of citizens in india that apply equally to all citizens regardless of their religion, gender and sexual orientation?', 'distractors': ['ASCII', 'American_Standard_Code_for_Information_Interchange', 'uniform civil code', 'samurai degree'], 'answer': 'uniform civil code'}]\n",
            "[{'question': 'What are the personal laws of different communities governed by?', 'distractors': ['religious texts', 'book', 'column', 'cookie'], 'answer': 'religious texts'}]\n",
            "[{'question': 'What does the bharatiya janata party defend?', 'distractors': ['sharia', 'Islamic_law', 'red', 'hudud'], 'answer': 'sharia'}]\n",
            "[{'question': 'What does personal law cover?', 'distractors': ['break', 'break_up', 'marriage, divorce', 'disassociate'], 'answer': 'marriage, divorce'}]\n",
            "[{'question': 'What type of law does the Uniform Civil Code cover?', 'distractors': ['X-linked_dominant_inheritance', 'X-linked_recessive_inheritance', 'accretion', 'inheritance'], 'answer': 'inheritance'}]\n"
          ]
        }
      ],
      "source": [
        "anvil.server.wait_forever()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}