Update app.py

app.py

@@ -1,91 +1,70 @@
 # #refer llama recipes for more info https://github.com/huggingface/huggingface-llama-recipes/blob/main/inference-api.ipynb
 # #huggingface-llama-recipes : https://github.com/huggingface/huggingface-llama-recipes/tree/main
-# import gradio as gr
-# from openai import OpenAI
+import gradio as gr
+from openai import OpenAI
 import os
 
 ACCESS_TOKEN = os.getenv("HF_TOKEN")
 
-# client = OpenAI(
-#     base_url="https://integrate.api.nvidia.com/v1",
-#     api_key=ACCESS_TOKEN,
-# )
+client = OpenAI(
+    base_url="https://integrate.api.nvidia.com/v1",
+    api_key=ACCESS_TOKEN,
+)
 
-# def respond(
-#     message,
-#     history: list[tuple[str, str]],
-#     system_message,
-#     max_tokens,
-#     temperature,
-#     top_p,
-# ):
-#     messages = [{"role": "system", "content": system_message}]
+def respond(
+    message,
+    history: list[tuple[str, str]],
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+):
+    messages = [{"role": "system", "content": system_message}]
 
-#     for val in history:
-#         if val[0]:
-#             messages.append({"role": "user", "content": val[0]})
-#         if val[1]:
-#             messages.append({"role": "assistant", "content": val[1]})
+    for val in history:
+        if val[0]:
+            messages.append({"role": "user", "content": val[0]})
+        if val[1]:
+            messages.append({"role": "assistant", "content": val[1]})
 
-#     messages.append({"role": "user", "content": message})
+    messages.append({"role": "user", "content": message})
 
-#     response = ""
+    response = ""
     
-#     for message in  client.chat.completions.create(
-#         model="nvidia/llama-3.1-nemotron-70b-instruct",
-#         max_tokens=max_tokens,
-#         stream=True,
-#         temperature=temperature,
-#         top_p=top_p,
-#         messages=messages,
-#     ):
-#         token = message.choices[0].delta.content
+    for message in  client.chat.completions.create(
+        model="nvidia/llama-3.1-nemotron-70b-instruct",
+        max_tokens=max_tokens,
+        stream=True,
+        temperature=temperature,
+        top_p=top_p,
+        messages=messages,
+    ):
+        token = message.choices[0].delta.content
         
-#         response += token
-#         yield response
+        response += token
+        yield response
         
-# chatbot = gr.Chatbot(height=600)
+chatbot = gr.Chatbot(height=600)
 
-# service = gr.ChatInterface(
-#     respond,
-#     additional_inputs=[
-#         gr.Textbox(value="", label="Системный промпт"),
-#         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Максимальная длина ответа"),
-#         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Температура"),
-#         gr.Slider(
-#             minimum=0.1,
-#             maximum=1.0,
-#             value=0.95,
-#             step=0.05,
-#             label="top_p",
-#         ),
+service = gr.ChatInterface(
+    respond,
+    additional_inputs=[
+        gr.Textbox(value="", label="Системный промпт"),
+        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Максимальная длина ответа"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Температура"),
+        gr.Slider(
+            minimum=0.1,
+            maximum=1.0,
+            value=0.95,
+            step=0.05,
+            label="top_p",
+        ),
         
-#     ],
-#     fill_height=True,
-#     chatbot=chatbot,
-#     theme=gr.themes.Soft(),
-# )
-# if __name__ == "__main__":
-#     service.launch()
-
-
-from openai import OpenAI
-
-client = OpenAI(
-  base_url = "https://integrate.api.nvidia.com/v1",
-  api_key = ACCESS_TOKEN
+    ],
+    fill_height=True,
+    chatbot=chatbot,
+    theme=gr.themes.Soft(),
 )
-
-completion = client.chat.completions.create(
-  model="nvidia/llama-3.1-nemotron-70b-instruct",
-  messages=[{"role":"user","content":"Write a limerick about the wonders of GPU computing."}],
-  temperature=0.5,
-  top_p=1,
-  max_tokens=1024,
-  stream=True
-)
-
-for chunk in completion:
-  if chunk.choices[0].delta.content is not None:
-    print(chunk.choices[0].delta.content, end="")
+if __name__ == "__main__":
+    service.launch()