Switch to Mistral model

README.md

@@ -8,6 +8,9 @@ sdk_version: 3.23.0
 app_file: app.py
 pinned: false
 duplicated_from: joaogante/transformers_streaming
+
+preload_from_hub:
+  - helenai/mistralai-Mistral-7B-Instruct-v0.2-ov
 ---
 
 # Environment

app.py

@@ -3,23 +3,24 @@ import subprocess
 from threading import Thread
 
 import gradio as gr
-from optimum.intel.openvino import OVModelForSeq2SeqLM
+from optimum.intel.openvino import OVModelForCausalLM
 from transformers import AutoTokenizer, TextIteratorStreamer
 
 result = subprocess.run(["lscpu"], text=True, capture_output=True)
 pprint.pprint(result.stdout)
 
-# original_model_id = "declare-lab/flan-alpaca-xl"
-original_model_id = "declare-lab/flan-alpaca-large"
-model_id = f"helenai/{original_model_id.replace('/','-')}-ov"
+original_model_id = "mistralai/Mistral-7B-Instruct-v0.2"
+model_id = "helenai/mistralai-Mistral-7B-Instruct-v0.2-ov"
 
-model = OVModelForSeq2SeqLM.from_pretrained(model_id)
+model = OVModelForCausalLM.from_pretrained(model_id)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 
 
 def run_generation(user_text, top_p, temperature, top_k, max_new_tokens):
-    # Get the model and tokenizer, and tokenize the user text.
-    model_inputs = tokenizer([user_text], return_tensors="pt")
+    # message = [{"role": "user", "content": "You are a helpful assistant"}, {"role": "assistant", "content": "How can I help?"}, {"role":"user", "content":user_text}]
+    message = [{"role": "user", "content": user_text}]
+
+    model_inputs = tokenizer.apply_chat_template(message, return_tensors="pt", return_dict=True)
 
     # Start generation on a separate thread, so that we don't block the UI. The text is pulled from the streamer
     # in the main thread. Adds timeout to the streamer to handle exceptions in the generation thread.
@@ -65,7 +66,6 @@ with gr.Blocks() as demo:
     with gr.Row():
         with gr.Column(scale=4):
             user_text = gr.Textbox(
-                placeholder="Write an email about an alpaca that likes flan",
                 label="User input",
             )
             model_output = gr.Textbox(label="Model output", lines=10, interactive=False)
@@ -117,3 +117,5 @@ with gr.Blocks() as demo:
     )
 
     demo.queue(max_size=32).launch(enable_queue=True, server_name="0.0.0.0")
+    # For local use:
+    # demo.launch(server_name="0.0.0.0")