GPEW_Final_Submission.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YWQ8JTN843it",
        "outputId": "2475d9c0-a7d5-4245-bae6-e17adfe778c8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: python-docx in /usr/local/lib/python3.10/dist-packages (0.8.11)\n",
            "Requirement already satisfied: lxml>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from python-docx) (4.9.3)\n"
          ]
        }
      ],
      "source": [
        "## Installations\n",
        "\n",
        "!pip install -q openai\n",
        "!pip install -q langchain\n",
        "!pip install -q unstructured\n",
        "!pip install -q tiktoken\n",
        "!pip install -q faiss-cpu\n",
        "!pip install python-docx"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Get the openai API key\n",
        "import openai\n",
        "from dotenv import load_dotenv, find_dotenv\n",
        "\n",
        "_ = load_dotenv(find_dotenv()) # read local .env file\n",
        "\n",
        "openai.api_key  = os.getenv('OPENAI_API_KEY')"
      ],
      "metadata": {
        "id": "s585-TPL5CXC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## Use this code if you want to export the AI message as a markdown word document\n",
        "\n",
        "from IPython.display import display, Markdown\n",
        "from docx import Document\n",
        "\n",
        "def save_as_word_document(text, filename):\n",
        "    document = Document()\n",
        "    document.add_paragraph(text)\n",
        "    document.save(filename)\n",
        "\n",
        "def disp_markdown(text):\n",
        "    display(Markdown(text))\n",
        "    save_as_word_document(text, \"project_proposal_german.docx\")"
      ],
      "metadata": {
        "id": "RpQY1IAN5km1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import langchain\n",
        "langchain.debug = False\n",
        "\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.document_loaders import UnstructuredFileLoader\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "from langchain.vectorstores import FAISS\n",
        "from langchain.chains import RetrievalQA\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain.chains import RetrievalQA\n",
        "\n",
        "\n",
        "# Load Data\n",
        "loader = UnstructuredFileLoader(\"raw_text.docx\")\n",
        "raw_documents = loader.load()\n",
        "\n",
        "# Split text into overlapping chuncks\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=15000,\n",
        "                                               chunk_overlap=8500,\n",
        "                                               length_function=len,\n",
        "                                               add_start_index=True)\n",
        "\n",
        "\n",
        "documents = text_splitter.split_documents(raw_documents)\n",
        "\n",
        "# Embedd the data\n",
        "embeddings = OpenAIEmbeddings(openai_api_key=openai.api_key)\n",
        "docsearch = FAISS.from_documents(documents, embeddings)\n",
        "\n",
        "# Set the model\n",
        "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo-16k\",\n",
        "                 openai_api_key=openai.api_key,\n",
        "                 temperature=0.0,\n",
        "                 max_tokens=240)\n",
        "\n",
        "# Set up the retrieval function\n",
        "qa_chain = RetrievalQA.from_chain_type(llm,\n",
        "                                       chain_type=\"refine\",\n",
        "                                       retriever=docsearch.as_retriever())"
      ],
      "metadata": {
        "id": "3kXzmIbl6EYI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# a list of \"questions\", each \"question\" corresponds to a section and its specific instructions\n",
        "\n",
        "sections = [\"\"\" I. Titel des FuE-Vorhabens:\n",
        "                  (Provide a succinct technical and numerical description and/or specifications of the project that captures its MAIN innovative core.\n",
        "\n",
        "                    Bear in mind the following:\n",
        "\n",
        "                    - Use specific technical language and must include any relevant parameters, numerical data, or statistics to accentuate new methodologies and intended results.\n",
        "                    - Aim for a detailed, comprehensive, and well-articulated response reflecting the breadth and depth of our research.\n",
        "                    - DO NOT INCLUDE THE SECTION HEADER.\n",
        "                    - Keep your response within 800 CHARACTERS and incorporate target parameters in absolute values.\n",
        "                    - Avoid mentioning information not directly related to the technical specifics and goals of the project.)\"\"\", #220\n",
        "\n",
        "            \"\"\" Start der FuE-Arbeiten (TT.MM.JJJJ):\n",
        "                 (Mention the start date of the research and development work.)\n",
        "\n",
        "               Ende der FuE-Arbeiten (TT.MM.JJJJ):\n",
        "                 (Mention the projected end date of the research and development work.)\"\"\" ,\n",
        "\n",
        "            \"\"\" II. Forschungszweig Hauptkategorie/Unterkategorie:\n",
        "                  (Specify the main and sub-category of the research discipline your project falls under.)\"\"\",\n",
        "\n",
        "            \"\"\"\n",
        "      III. Inhaltlich/Fachliche Angaben:\n",
        "        ( Based on the provided context, formulate a cohesive paragraph detailing the current state of art/technology and its limitations relevant to the project's technical description.\n",
        "         Highlight how the project extends or builds upon this state, showcasing its innovation and potential impact.\n",
        "         Specify the project's goal, emphasizing its innovative core and associated technical terms.\n",
        "         Detail the expected outcome or result, illustrating its contribution to advancing the field.\n",
        "\n",
        "        Bear in mind the following:\n",
        "\n",
        "          - Use specific technical language and must include any relevant parameters, numerical data, or statistics to accentuate new methodologies and intended results.\n",
        "          - Aim for a detailed, comprehensive, and well-articulated response reflecting the breadth and depth of our research.\n",
        "          - DO NOT INCLUDE THE SECTION HEADER.\n",
        "          - Keep your response within 800 CHARACTERS and incorporate target parameters in absolute values.\n",
        "          - Avoid mentioning information not directly related to the technical specifics and goals of the project.)\"\"\",\n",
        "\n",
        "      \"\"\"\n",
        "      IV. Beschreibung der Arbeiten (alle Tätigkeitsformen: Eigenbetriebliche\n",
        "          Forschung/Auftragsforschung/Kooperationsforschung)\n",
        "          (Based on the provided context, formulate a cohesive paragraph detailing all the steps, tasks and methods that will be employed to accomplish the project's objective. Make sure you detail the tools and methodologies that will be used to implement each step.\n",
        "\n",
        "            Bear in mind the following:\n",
        "\n",
        "            - Use specific technical language and must include any relevant parameters, numerical data, or statistics to accentuate new methodologies and intended results.\n",
        "            - Aim for a succinct, comprehensive, and well-articulated response reflecting the breadth and depth of our research.\n",
        "            - DO NOT INCLUDE THE SECTION HEADER AND INTRODUCTORY SENTENCES.\n",
        "            - Keep your response within 800 CHARACTERS and incorporate target parameters in absolute values.\n",
        "            - Avoid mentioning information not directly related to the technical specifics and goals of the project.)\"\"\",\n",
        "\n",
        "\n",
        "\"\"\"   V. Zielt das Vorhaben auf ein Produkt, Produktionsverfahren, eine Produktionslinie, eine Dienstleistung oder eine wissenschaftliche Methodik ab, das/die eine deutliche Weiterentwicklung/Neuheit in Bezug auf den betreffenden Wirtschaftszweig darstellt?\n",
        "          (In this section, your task is to:\n",
        "          1. Specify the target sector/product category and explain how your project differentiates from the existing state of the art within that industry.\n",
        "          2. Describe any new scientific/technical methods, approaches, or procedures that will be utilized in your project.\n",
        "          3. Provide examples of experiments, field studies, observations, cohort studies, or any other relevant activities.\n",
        "          4. Highlight approximately two parameters of differentiation to showcase how your project aims to surpass the state of the art in terms of innovation and advancement.\n",
        "\n",
        "          Note: Remember to use specific details to describe your project and its differences from existing solutions.\n",
        "          This will demonstrate the innovative nature of your project and show how it aligns with the FRASCATI criteria.\n",
        "          Your responses should be detailed, , well-explained but straight to the point. Don't include the section header.\n",
        "          Ensure your response MUST NOT EXCEED 800 CHARACTERS, and it must include the target parameters in the form of absolute value)\"\"\",\n",
        "\n",
        "            \"\"\"\n",
        "      VI. Besteht ein konkreter Bezug des Vorhabens zu bestehenden Produkten, Produktionslinien, Produktionsverfahren, Dienstleistungen oder bereits etablierter wissenschaftlicher Methodik in Ihrem Unternehmen?\n",
        "        (Based on the provided context:\n",
        "          1. Describe the technologies, methods or solutions currently in use at the company that relate to the proposed project.\n",
        "          2. Outline the unique functionality that distinguishes the proposed project from the existing solutions.\n",
        "          3. Detail how the to-be-developed product, process, or service varies from those your company already offers.\n",
        "\n",
        "        Bear in mind the following:\n",
        "\n",
        "        - Use specific technical language and must include any relevant parameters, numerical data, or statistics to accentuate new methodologies and intended results.\n",
        "        - Aim for a succinct, comprehensive, and well-articulated response reflecting the breadth and depth of our research.\n",
        "        - DO NOT INCLUDE THE SECTION HEADER AND INTRODUCTORY SENTENCES.\n",
        "        - Keep your response within 800 CHARACTERS and incorporate target parameters in absolute values.\n",
        "        - Avoid mentioning information not directly related to the technical specifics and goals of the project.) \"\"\",\n",
        "\n",
        "            \"\"\"\n",
        "      VII. Erläuterung der wissenschaftlichen und/oder technischen Risiken bei der Umsetzung des Vorhabens\n",
        "          (Based on the provided context, formulate a cohesive paragraph explaining all potential risks - engineering, natural, social, or humanities - that could hinder the project.\n",
        "           Describe scientific or technical uncertainties or challenges that might lead to failure.\n",
        "\n",
        "        Bear in mind the following:\n",
        "\n",
        "        - Use specific technical language and must include any relevant parameters, numerical data, or statistics to accentuate new methodologies and intended results.\n",
        "        - Aim for a succinct, comprehensive, and well-articulated response reflecting the breadth and depth of our research.\n",
        "        - DO NOT INCLUDE THE SECTION HEADER AND INTRODUCTORY SENTENCES.\n",
        "        - Keep your response within 800 CHARACTERS and incorporate target parameters in absolute values.\n",
        "        - Avoid mentioning information not directly related to the technical specifics and goals of the project.)\"\"\",\n",
        "\n",
        "            \"\"\"\n",
        "      VIII. Tätigkeitsform:\n",
        "          (In this section, your task is to articulate a comprehensive analysis of whether the activities and responsibilities of your project fall under in-house research and development, contract research, or both.\n",
        "\n",
        "            For reference:\n",
        "            - In-house research and development: refers to activities conducted by the staff of the organization itself, often implying that the organization has control over the research agenda, process, and results.\n",
        "            - Contract research, on the other hand, refers to activities where an organization contracts another party (usually another research institution or a specialized research company) to carry out specific R&D tasks.\n",
        "              This often happens when the commissioning organization lacks certain skills, resources, or capacity to perform those tasks internally.\n",
        "\n",
        "            Note: Utilize specific technical terminology to emphasize the novel technical approaches.\n",
        "            Your responses should be thorough, detailed, and well-explained responses that reveal the depth and breadth of our research activities.\n",
        "            DO NOT INCLUDE THE SECTION HEADER. Ensure your response DOES NOT EXCEED 800 CHARACTERS and includes target parameters in absolute values.)\"\"\",\n",
        "\n",
        "      \"\"\"\n",
        "      X. Beschreibung der in Auftrag gegebenen FuE-Arbeiten und der damit verbundenen Ziele\n",
        "        (In this section, your task is to Describe the research and development work commissioned for the project and its objectives.\n",
        "\n",
        "          Note: Utilize specific technical terminology to emphasize the novel technical approaches.\n",
        "          Your responses should be thorough, detailed, and well-explained responses that reveal the depth and breadth of our research activities.\n",
        "          DO NOT INCLUDE THE SECTION HEADER. Ensure your response DOES NOT EXCEED 800 CHARACTERS and includes target parameters in absolute values.)\"\"\",\n",
        "\n",
        "            \"\"\"\n",
        "      XI. Verschlagwortung\n",
        "        Extract all the relevant technical and scientific keywords mentioned in the provided context. \"\"\",\n",
        "\n",
        "            \"\"\"\n",
        "            XII. Verwertung\n",
        "         (Based on the provided context, formulate a cohesive and succinct explanation of how the project's outcomes will be utilized and disseminated (to end users and the scientific community).\n",
        "\n",
        "        Bear in mind the following:\n",
        "\n",
        "        - Use specific technical language and must include any relevant parameters, numerical data, or statistics to accentuate new methodologies and intended results.\n",
        "        - Aim for a succinct, comprehensive, and well-articulated response reflecting the breadth and depth of our research.\n",
        "        - DO NOT INCLUDE THE SECTION HEADER AND INTRODUCTORY SENTENCES.\n",
        "        - Keep your response within 800 CHARACTERS and incorporate target parameters in absolute values.\n",
        "        - Avoid mentioning information not directly related to the technical specifics and goals of the project.)\"\"\"\n",
        "]"
      ],
      "metadata": {
        "id": "_Ej5Z4iV6NVY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Build prompt (System message + \"question\")\n",
        "from langchain.prompts import PromptTemplate\n",
        "template = \"\"\"\n",
        "As a project manager in an R&D department, your task is to construct a comprehensive project description in German based on the provided German context.\n",
        "The proposal should be innovative, align with the FRASCATI criteria, and be suitable for submission to the German tax R&D subsidy grant scheme.\n",
        "The proposal should reflect the provided German context, encapsulating specific objectives, prospective impact, and significant advancements in technology, methodology, or application in the relevant field.\n",
        "\n",
        "Respond in German while considering all the following points:\n",
        "\n",
        "-  Preserve all critical information, including technical terms, dates, statistics, and numerical data from the context.\n",
        "-  Emphasize the uniqueness and novelty of the project, highlighting its potential economic, societal, or environmental benefits.\n",
        "-  Provide comprehensive and accurate answers, refraining from inventing information. Always refer back to the given context.\n",
        "-  Utilize technical and scientific language whenever suitable, specifying the methodologies used.\n",
        "-  Your responses should be to the point and in the form of paragraphs and MUST NOT EXCEED 800 CHARACTERS.\n",
        "\n",
        "{context}\n",
        "\n",
        "Question:\n",
        "\n",
        "      III. Inhaltlich/Fachliche Angaben:\n",
        "        ( Based on the provided context, formulate a cohesive paragraph detailing the current state of art/technology and its limitations relevant to the project's technical description.\n",
        "         Highlight how the project extends or builds upon this state, showcasing its innovation and potential impact.\n",
        "         Specify the project's goal, emphasizing its innovative core and associated technical terms.\n",
        "         Detail the expected outcome or result, illustrating its contribution to advancing the field.\n",
        "\n",
        "        Bear in mind the following:\n",
        "\n",
        "          - Use specific technical language and must include any relevant parameters, numerical data, or statistics to accentuate new methodologies and intended results.\n",
        "          - Aim for a detailed, comprehensive, and well-articulated response reflecting the breadth and depth of our research.\n",
        "          - DO NOT INCLUDE THE SECTION HEADER.\n",
        "          - Keep your response within 800 CHARACTERS and incorporate target parameters in absolute values.\n",
        "          - Avoid mentioning information not directly related to the technical specifics and goals of the project.)\n",
        "\"\"\"\n",
        "\n",
        "# Run chain\n",
        "result = qa_chain({\"query\": template})\n",
        "result['result']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "id": "SyY881yV6TEA",
        "outputId": "98d8f4b6-ad81-4cac-8c0f-94f8a15ffa60"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Derzeit werden im Bereich der genetischen Gesundheitsbewertung von Haustieren vor allem Einzeltests und rassenspezifische Produkte eingesetzt. Diese Tests sind jedoch begrenzt und können nicht das gesamte genetische Markup eines Tieres untersuchen. Unser Projekt zielt darauf ab, diese Lücke zu schließen und eine innovative Methode einzuführen, die auf dem Next Generation Sequencing (NGS) basiert. Durch die Anwendung von NGS können wir das gesamte Genom eines Tieres analysieren und genetische Varianten wie Insertionen, Deletionen und Inversionen identifizieren. Unser Ziel ist es, für die Spezies Hund, Katze und Pferd ein umfassendes Testpaket zu entwickeln, das eine Vielzahl von Erbkrankheiten abdeckt. Wir streben an, mehr als 60 Prozent der bekannten Erbkrankheiten in jeder Spezies zu erfassen. Das Ergebnis wird ein Testkit sein, das es Tierbesitzern, Züchtern und Tierär'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 94
        }
      ]
    }
  ]
}