Update app.py

app.py

@@ -73,11 +73,12 @@ def respond(
     temperature,
     top_p,
 ):
-    model= model.to('cuda')
+    
     messages = [{"role": "user", "content": "Hello, how are you?"}]
     input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to('cuda')
 ## <BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello, how are you?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
     # with autocast():
+    model= model.to('cuda')
     gen_tokens = model.generate(
     input_ids, 
     max_new_tokens=100,