mistral-7b-inference.ipynb

{"cells":[{"cell_type":"markdown","metadata":{},"source":["## Model Inference"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["pip install --upgrade numpy==1.22.0\n","pip install --upgrade scipy==1.7.3"]},{"cell_type":"code","execution_count":1,"metadata":{"execution":{"iopub.execute_input":"2023-11-23T09:44:57.334122Z","iopub.status.busy":"2023-11-23T09:44:57.333828Z","iopub.status.idle":"2023-11-23T09:45:50.736473Z","shell.execute_reply":"2023-11-23T09:45:50.735253Z","shell.execute_reply.started":"2023-11-23T09:44:57.334095Z"},"trusted":true},"outputs":[],"source":["!pip install -q -U transformers\n","!pip install -q -U accelerate\n","!pip install -q -U bitsandbytes>=0.37.0"]},{"cell_type":"code","execution_count":2,"metadata":{"execution":{"iopub.execute_input":"2023-11-23T09:46:38.495833Z","iopub.status.busy":"2023-11-23T09:46:38.494893Z","iopub.status.idle":"2023-11-23T09:46:54.899327Z","shell.execute_reply":"2023-11-23T09:46:54.898342Z","shell.execute_reply.started":"2023-11-23T09:46:38.495798Z"},"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n","  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"]}],"source":["from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, pipeline\n","import torch\n","\n","bnb_config = BitsAndBytesConfig(\n","    load_in_4bit=True,\n","    bnb_4bit_quant_type=\"nf4\",\n","    bnb_4bit_use_double_quant=True,\n",")"]},{"cell_type":"code","execution_count":3,"metadata":{"execution":{"iopub.execute_input":"2023-11-23T09:46:58.135224Z","iopub.status.busy":"2023-11-23T09:46:58.134561Z","iopub.status.idle":"2023-11-23T09:49:25.502068Z","shell.execute_reply":"2023-11-23T09:49:25.501231Z","shell.execute_reply.started":"2023-11-23T09:46:58.135191Z"},"trusted":true},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"88a46f9c8a624a4090dedee5db075657","version_major":2,"version_minor":0},"text/plain":["Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"}],"source":["model_name = \"/kaggle/input/mistral/pytorch/7b-v0.1-hf/1\"\n","\n","tokenizer = AutoTokenizer.from_pretrained(model_name)\n","tokenizer.pad_token = tokenizer.eos_token\n","\n","model = AutoModelForCausalLM.from_pretrained(\n","    model_name,\n","    load_in_4bit=True,\n","    quantization_config=bnb_config,\n","    torch_dtype=torch.bfloat16,\n","    device_map=\"auto\",\n","    trust_remote_code=True,\n",")"]},{"cell_type":"code","execution_count":4,"metadata":{"execution":{"iopub.execute_input":"2023-11-23T10:16:39.270985Z","iopub.status.busy":"2023-11-23T10:16:39.270054Z","iopub.status.idle":"2023-11-23T10:16:39.275704Z","shell.execute_reply":"2023-11-23T10:16:39.274762Z","shell.execute_reply.started":"2023-11-23T10:16:39.270950Z"},"trusted":true},"outputs":[],"source":["pipe = pipeline(\n","    \"text-generation\", \n","    model=model, \n","    tokenizer = tokenizer, \n","    torch_dtype=torch.bfloat16, \n","    device_map=\"auto\"\n",")"]},{"cell_type":"code","execution_count":8,"metadata":{"execution":{"iopub.execute_input":"2023-11-23T10:22:53.818034Z","iopub.status.busy":"2023-11-23T10:22:53.817193Z","iopub.status.idle":"2023-11-23T10:23:02.413687Z","shell.execute_reply":"2023-11-23T10:23:02.412696Z","shell.execute_reply.started":"2023-11-23T10:22:53.817999Z"},"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"]},{"name":"stdout","output_type":"stream","text":["What is Neural Network?\n"," Neural Network is a mathematical model of a biological nervous system.\n","\n","\n","## What is deep learning?\n","\n","Deep learning is a machine learning technique that emulates the workings of the human brain in processing data and creating patterns.\n","\n","\n","## How does a neural network work?\n","\n","A neural network is a mathematical model of a biological nervous system. It consists of a series of interconnected nodes, or neurons, which can be thought of as a way of connecting input\n"]}],"source":["prompt = \"What is Neural Network?\"\n","\n","sequences = pipe(\n","    prompt,\n","    do_sample=True,\n","    max_new_tokens=100, \n","    temperature=0.7, \n","    top_k=50, \n","    top_p=0.95,\n","    num_return_sequences=1,\n",")\n","\n","print(sequences[0]['generated_text'])"]}],"metadata":{"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"isSourceIdPinned":true,"modelInstanceId":3898,"sourceId":5110,"sourceType":"modelInstanceVersion"},{"modelInstanceId":3899,"sourceId":5111,"sourceType":"modelInstanceVersion"}],"dockerImageVersionId":30559,"isGpuEnabled":true,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"}},"nbformat":4,"nbformat_minor":4}